level=info ts=2026-05-04T11:51:49.415929532Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-04T11:51:49.415957492Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-04T11:51:49.416276851Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-04T11:51:49.417008261Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-04T11:51:49.417380726Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T11:51:49.417445415Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.60 inf=eth0
level=info ts=2026-05-04T11:51:49.418672818Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-1d2a49ff
ts=2026-05-04T11:51:49.418782397Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-04T11:51:49.418799294Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.60 advertisePort=7946
level=debug ts=2026-05-04T11:51:49.41882109Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.60 advertisePort=7946
level=info ts=2026-05-04T11:51:49.418980423Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-04T11:51:49.421510379Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T11:51:49.421543972Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-04T11:51:49.423521901Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-04T11:51:49.423939659Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-04T11:51:49.423956269Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-04T11:51:49.423954425Z
level=debug ts=2026-05-04T11:51:49.42397516Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-04T11:51:49.423997529Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-04T11:51:49.424016011Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-04T11:51:49.424024054Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-04T11:51:49.424022406Z
level=debug ts=2026-05-04T11:51:49.424037709Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-04T11:51:49.424214321Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T11:51:49.426049292Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-04T11:51:49.426125082Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.60 inf=eth0
ts=2026-05-04T11:51:49.426638063Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T11:51:49.426661238Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:51:49.426671036Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.693564ms
level=error ts=2026-05-04T11:51:49.426679279Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-04T11:51:49.426685933Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-04T11:51:49.428839106Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:49.428852127Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T11:51:49.439622125Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-04T11:51:49.439626515Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-04T11:51:49.439638876Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-04T11:51:49.439644476Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=info ts=2026-05-04T11:51:49.439660875Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-04T11:51:49.439810772Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-04T11:51:49.439656758Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-04T11:51:49.439969296Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-04T11:51:49.439988358Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-04T11:51:49.439994873Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-04T11:51:49.440250503Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-04T11:51:49.440258708Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-04T11:51:49.440262437Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-05-04T11:51:49.440255657Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-04T11:51:49.440273461Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-04T11:51:49.440274792Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-04T11:51:49.44027864Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-05-04T11:51:49.440277022Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-05-04T11:51:49.440302861Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-04T11:51:49.440346432Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-04T11:51:49.440350668Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-04T11:51:49.440353743Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-05-04T11:51:49.440356361Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-04T11:51:49.440444673Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=info ts=2026-05-04T11:51:49.440449452Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-04T11:51:49.440455193Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-04T11:51:49.440464878Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T11:51:49.440504254Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.211µs success=true
level=info ts=2026-05-04T11:51:49.440516054Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-04T11:51:49.440560546Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=104.947µs errors=false
level=info ts=2026-05-04T11:51:49.440569917Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-04T11:51:49.440690735Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=235.667µs errors=false
level=info ts=2026-05-04T11:51:49.440702559Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-04T11:51:49.440707894Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=253.284µs
level=info ts=2026-05-04T11:51:49.440753371Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-04T11:51:49.441000304Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-04T11:51:49.441020002Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-04T11:51:49.441057845Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=3.247687336s
level=debug ts=2026-05-04T11:51:49.441114218Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-04T11:51:49.44112362Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-05-04T11:51:49.441135488Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-04T11:51:49.441144597Z caller=loki.go:615 msg="Loki started" startup_time=88.991518ms
level=debug ts=2026-05-04T11:51:49.441166944Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-04T11:51:51.114553258Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:51.114577163Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:51:54.44124217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:51:55.035916201Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:55.035939131Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:51:59.432749835Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:51:59.432771566Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:51:59.441960914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:52:04.441601238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:52:05.150596016Z caller=logging.go:137 msg="GET /ready (503) 105.664µs"
level=warn ts=2026-05-04T11:52:09.441373421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:52:12.035283205Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.55:7946: dial tcp 10.128.0.55:7946: connect: connection refused"
ts=2026-05-04T11:52:12.036492252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.56:7946"
ts=2026-05-04T11:52:12.038923806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.59:7946"
ts=2026-05-04T11:52:12.040368032Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.56:7946"
ts=2026-05-04T11:52:12.041768626Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.58:7946: dial tcp 10.129.0.58:7946: connect: connection refused"
level=info ts=2026-05-04T11:52:12.041780636Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=22.61509073s
level=warn ts=2026-05-04T11:52:14.441568597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:52:15.149595282Z caller=logging.go:137 msg="GET /ready (503) 78.752µs"
level=warn ts=2026-05-04T11:52:19.442181569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:52:19.442176351Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T11:52:19.442224735Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-04T11:52:19.442235745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:52:22.785031503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:52:24.44159436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:52:24.985422281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=warn ts=2026-05-04T11:52:29.441399279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:52:34.441667331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:52:39.44144103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:52:40.313433831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:52:44.441359065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:52:47.499208332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:53948"
level=debug ts=2026-05-04T11:52:49.423444409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:52:49.42448746Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T11:52:49.441874114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T11:52:49.441877416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:52:49.441891802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:52:49.441900067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:52:53.082705657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T11:52:53.763527899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:35304"
level=warn ts=2026-05-04T11:52:54.441374261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:52:54.987648665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=warn ts=2026-05-04T11:52:59.441837795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:53:03.048436312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:53:04.44129975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:53:09.441245885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:53:14.441362925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:53:16.312058245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:39442"
level=info ts=2026-05-04T11:53:19.441775714Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:53:19.441812732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:53:19.441821322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:53:19.441806545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:53:23.767001608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59772"
level=warn ts=2026-05-04T11:53:24.442233357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:53:24.991036166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:53:25.9907779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:53:29.441290108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:53:31.894688054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:53:34.441666024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:53:39.441756684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:53:44.44169474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:53:49.423246182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:53:49.42430955Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-04T11:53:49.441706227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:53:49.441706495Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:53:49.44173031Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:53:49.441739054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:53:51.24558288Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T11:53:51.245629774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:53:51.255206566Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T11:53:52.449030511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-04T11:53:53.770638551Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:50652"
level=warn ts=2026-05-04T11:53:54.441379618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:53:54.994153511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=warn ts=2026-05-04T11:53:59.442107416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:54:04.44144826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:54:09.44177463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:54:10.96792609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:54:10.975365618Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T11:54:14.441963577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:54:19.441909962Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T11:54:19.441911496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:54:19.441940444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:54:19.441948383Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:54:23.449869086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T11:54:23.451671199Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T11:54:24.441991059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:54:24.995907301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T11:54:26.922951892Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:54:29.441844152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:54:34.441837418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:54:39.441978099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:54:44.441716131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:54:47.510268646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49490"
level=info ts=2026-05-04T11:54:47.860564924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:54:47.868817307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T11:54:49.42387491Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:54:49.424943823Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T11:54:49.441142397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:54:49.441160654Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:54:49.441168902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:54:49.441231649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:54:51.482921796Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:54:54.441585657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:54:54.997012719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=warn ts=2026-05-04T11:54:59.441229011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:55:04.44158481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:55:09.441309335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:55:10.672314861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:55:14.441534856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:55:17.511201304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:59930"
level=info ts=2026-05-04T11:55:19.441526134Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:55:19.441554571Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:55:19.441565221Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:55:19.441617189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:55:24.442115099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:55:24.999383741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:55:25.647455938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:55:29.4420946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:55:34.441705939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:55:39.441960911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:55:39.456921699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:55:39.46505035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T11:55:44.441317089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:55:49.4235643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:55:49.424626878Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T11:55:49.441966148Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:55:49.441981379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:55:49.441989661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:55:49.441995418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:55:53.78488179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:50454"
level=warn ts=2026-05-04T11:55:54.441399922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:55:55.002811239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:55:59.192668312Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:55:59.441708108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:56:04.441805727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:56:09.441257398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:56:14.442214515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:56:16.325991653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46762"
level=info ts=2026-05-04T11:56:19.441488874Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T11:56:19.441503596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:56:19.441518391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:56:19.441526849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-04T11:56:23.787696717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:38162"
level=debug ts=2026-05-04T11:56:24.065178777Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-04T11:56:24.442128982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:56:25.005727642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:56:28.413619712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:56:29.441651976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:56:31.427434035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:56:31.436834864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T11:56:34.441747671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:56:39.441556515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T11:56:44.442159219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:56:49.424178019Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:56:49.425210837Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T11:56:49.441479603Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:56:49.441493745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:56:49.441488555Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T11:56:49.441501241Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:56:49.441542526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:56:49.441620653Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-04T11:56:49.4416285Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T11:56:49.441642678Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.456µs success=true
level=debug ts=2026-05-04T11:56:49.441828336Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-04T11:56:53.123715473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T11:56:53.797342234Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T11:56:54.441714382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:56:55.009054442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:56:58.151840836Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-04T11:56:59.442205371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:02.507219363Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-04T11:57:04.441395102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:06.862127768Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T11:57:09.441660654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:11.216741053Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-04T11:57:14.441232589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:15.571898177Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T11:57:17.279586684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:57:17.286600763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T11:57:17.518340996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44502"
level=warn ts=2026-05-04T11:57:19.441288425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:57:19.441294836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:57:19.441320771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:57:19.441331318Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:57:19.926326332Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-04T11:57:23.41351391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-04T11:57:24.281039915Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-04T11:57:24.441410309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:57:25.012366246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:57:27.124428662Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T11:57:28.635640326Z caller=checkpoint.go:372 msg="writing series" size="6.4 kB"
level=warn ts=2026-05-04T11:57:29.441814429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:32.990883107Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=warn ts=2026-05-04T11:57:34.44212041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:37.345592147Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T11:57:39.441545616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:41.700351931Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-04T11:57:41.700379067Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-04T11:57:44.441994664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:46.055246777Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-04T11:57:49.424258795Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T11:57:49.424260968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:57:49.44160688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T11:57:49.441610926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:49.441627272Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:57:49.441635477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:57:50.409941245Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=warn ts=2026-05-04T11:57:54.442101009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:57:54.764745068Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-04T11:57:55.014070225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T11:57:56.007002867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T11:57:59.11945539Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=warn ts=2026-05-04T11:57:59.441315012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:03.474583506Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T11:58:04.441935777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:07.829575019Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-04T11:58:07.967842295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:58:07.975303927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T11:58:09.44183147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:12.185158298Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T11:58:12.185179073Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=warn ts=2026-05-04T11:58:14.442241987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:16.539426707Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-04T11:58:17.438565575Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-04T11:58:19.442158145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T11:58:19.442171149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:19.442201309Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:58:19.442210548Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T11:58:20.893976218Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-04T11:58:24.441591983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:58:25.015172397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T11:58:25.248955514Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=warn ts=2026-05-04T11:58:29.441845975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:29.603349064Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=debug ts=2026-05-04T11:58:31.575020534Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T11:58:33.958852667Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-04T11:58:34.441380534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:38.3154973Z caller=checkpoint.go:372 msg="writing series" size="4.6 MB"
level=debug ts=2026-05-04T11:58:38.315520002Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=6
level=warn ts=2026-05-04T11:58:39.441348157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:42.667782876Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-04T11:58:44.441773084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:47.022613961Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T11:58:49.424252524Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:58:49.42426123Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T11:58:49.441599205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:58:49.441616574Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:58:49.441624025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:58:49.44164247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:58:51.377576231Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T11:58:54.441905629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:58:55.017759613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T11:58:55.733003068Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-04T11:58:58.653353726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T11:58:59.441645659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:00.087240293Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-04T11:59:01.255846651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:59:01.26324449Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T11:59:04.44198185Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-04T11:59:04.44197958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:08.796775931Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-04T11:59:09.441648136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:13.152058184Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T11:59:14.442108843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:17.494832884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-04T11:59:17.496298251Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T11:59:17.506341162Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-04T11:59:19.4418241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:59:19.441851316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:59:19.441859648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:59:19.441876001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:21.861422865Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-04T11:59:23.508257166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T11:59:23.807105727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:35344"
level=warn ts=2026-05-04T11:59:24.441586398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:59:25.020250054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T11:59:26.216604817Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-04T11:59:29.442218463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:30.570913499Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T11:59:34.441386565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:34.926308683Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-04T11:59:39.281422189Z caller=checkpoint.go:372 msg="writing series" size="1.4 kB"
level=warn ts=2026-05-04T11:59:39.441923247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:43.636061912Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-04T11:59:44.441198846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T11:59:45.756219277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T11:59:45.766486378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T11:59:47.990835037Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-04T11:59:49.051477641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T11:59:49.423426205Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T11:59:49.424491819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T11:59:49.441849025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T11:59:49.441863161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T11:59:49.441870555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T11:59:49.441904845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T11:59:52.345816841Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-04T11:59:54.441594993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T11:59:55.021752333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T11:59:56.700872869Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-04T11:59:59.441948727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:01.05531871Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-04T12:00:04.442198681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:05.409910397Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-04T12:00:09.441329977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:09.765386088Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-04T12:00:12.858076409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:00:14.120257451Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:00:14.442001108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:17.825379673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:00:18.474215619Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-04T12:00:18.474232573Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=23
level=info ts=2026-05-04T12:00:19.442058612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:00:19.442090087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:00:19.442098768Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:00:19.442116325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:22.829959687Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=warn ts=2026-05-04T12:00:24.441844745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:00:25.024125448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:00:27.184390186Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-04T12:00:29.441840788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:31.538743992Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-04T12:00:34.442030036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:00:34.642137393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:00:34.65017124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:00:34.650210358Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:00:34.650239586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:00:34.656716221Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:00:35.893537065Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-04T12:00:39.441319351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:40.249242164Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-04T12:00:44.44159977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:44.603325745Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-04T12:00:48.958372044Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=debug ts=2026-05-04T12:00:48.958392692Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-04T12:00:49.214169655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:00:49.224107911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:00:49.423377051Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:00:49.424442348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:00:49.425707306Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T12:00:49.42574381Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975439 n=1
level=debug ts=2026-05-04T12:00:49.425786796Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975439 n=1
level=info ts=2026-05-04T12:00:49.441798096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:00:49.441819109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:00:49.441827661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:00:49.441838446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:00:50.116117745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:00:52.112279958Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:00:53.313656756Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-04T12:00:53.816716116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52992"
level=warn ts=2026-05-04T12:00:54.44191482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:00:55.026781669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:00:57.668093244Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-04T12:00:59.441411982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:02.023452908Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:01:04.441752509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:06.377668245Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-04T12:01:09.441978111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:10.732234392Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-04T12:01:14.441839493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:15.087683947Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-04T12:01:19.441516008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:01:19.441515966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:19.441542004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:01:19.441550223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:01:19.44259987Z caller=checkpoint.go:384 msg="flushing series" totalSize="156 kB" series=6
level=info ts=2026-05-04T12:01:19.446804536Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-04T12:01:19.449765461Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007998052s
level=warn ts=2026-05-04T12:01:24.441540375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:24.601430509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:01:25.028566576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:01:28.591520891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:01:28.599859218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:01:29.441690279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:01:34.44153218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:01:39.441256381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:43.349557916Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:01:44.441331642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:01:47.539143868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43784"
level=debug ts=2026-05-04T12:01:49.423370273Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:01:49.424490012Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:01:49.44077113Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:01:49.440978495Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-04T12:01:49.440990483Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:01:49.441011783Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.362µs success=true
level=debug ts=2026-05-04T12:01:49.441260131Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-04T12:01:49.441283911Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:01:49.441296231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:01:49.441304846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:01:49.441313567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:01:53.796786925Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-04T12:01:53.813909171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:01:53.82266338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:53926"
level=warn ts=2026-05-04T12:01:54.441445315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:01:55.030719991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:01:58.151827133Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-04T12:01:59.44161668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:02.506768058Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-04T12:02:04.441924339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:06.860840764Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:02:09.441764692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:11.215788839Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-04T12:02:13.561392326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:02:13.571384619Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:02:14.441982922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:15.570866096Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:02:19.441514229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:02:19.441516031Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:02:19.441551523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:02:19.441566429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:02:19.92610924Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-04T12:02:23.413805017Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:02:23.82549299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42772"
level=debug ts=2026-05-04T12:02:24.280259415Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-04T12:02:24.44129521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:02:25.032343706Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:02:25.571861959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:02:28.635613822Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=warn ts=2026-05-04T12:02:29.441240353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:32.990349842Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=warn ts=2026-05-04T12:02:34.442109717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:37.345259448Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-04T12:02:37.345281341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-04T12:02:39.441328966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:41.700295772Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-04T12:02:44.441531365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:46.054586662Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
ts=2026-05-04T12:02:46.354868053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:57024"
level=debug ts=2026-05-04T12:02:49.424241872Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:02:49.424271448Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-04T12:02:49.441697467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:02:49.441693759Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:02:49.441719434Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:02:49.441727186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:02:50.4094529Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
ts=2026-05-04T12:02:53.829100748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:44348"
level=debug ts=2026-05-04T12:02:54.266722419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:02:54.442134982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:02:54.76472344Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
ts=2026-05-04T12:02:55.033118398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:02:59.119053337Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=warn ts=2026-05-04T12:02:59.442018131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:03:02.619262506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:03:02.628944572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:03:03.474134989Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:03:04.442491962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:03:07.829211556Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=debug ts=2026-05-04T12:03:09.375865181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:03:09.377597846Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T12:03:09.44191106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:03:12.184531643Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-04T12:03:12.184556233Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=8
level=warn ts=2026-05-04T12:03:14.442124178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:03:16.355929989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50860"
level=debug ts=2026-05-04T12:03:16.5390341Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-04T12:03:17.545098714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:39786"
level=info ts=2026-05-04T12:03:19.442107248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:03:19.442138316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:03:19.442148361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:03:19.442134683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:03:20.893660833Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-04T12:03:24.441719406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:03:25.035408049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:03:25.248154151Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-04T12:03:25.50290734Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:03:25.502953742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:03:25.511722242Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:03:29.442222407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:03:29.603672607Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-04T12:03:33.957824556Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-04T12:03:34.441640479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:03:37.68130561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:03:37.691611208Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:03:38.316221131Z caller=checkpoint.go:372 msg="writing series" size="4.7 MB"
level=debug ts=2026-05-04T12:03:38.316250744Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=6
level=warn ts=2026-05-04T12:03:39.441388347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:03:42.667423442Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-04T12:03:44.44164476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:03:46.357162922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51490"
level=debug ts=2026-05-04T12:03:47.021939379Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T12:03:47.373923253Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:03:47.546900604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43208"
level=debug ts=2026-05-04T12:03:49.423381455Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:03:49.424439036Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:03:49.442112264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:03:49.442138363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:03:49.442148467Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:03:49.442176584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:03:49.658896548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:03:49.669131296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:03:51.377417346Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
ts=2026-05-04T12:03:53.835777775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:55966"
level=warn ts=2026-05-04T12:03:54.441272479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:03:55.038321994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:03:55.732262718Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T12:03:59.441878179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:00.062725356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:04:00.086832858Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-04T12:04:04.441315442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:04.441379651Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:04:08.796706062Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=warn ts=2026-05-04T12:04:09.442152775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:13.151236033Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-04T12:04:14.441758241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:04:16.358701095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55002"
level=info ts=2026-05-04T12:04:16.815403385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:04:16.825427592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:04:17.506392096Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
ts=2026-05-04T12:04:17.54805191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:40458"
level=info ts=2026-05-04T12:04:19.44155941Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:04:19.441563584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:19.441584698Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:04:19.441592923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:04:21.86124538Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-05-04T12:04:21.861269815Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-05-04T12:04:24.442035371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:04:25.040310941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:04:26.216433734Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=debug ts=2026-05-04T12:04:27.960306722Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:04:29.441660939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:30.570902794Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T12:04:34.441352874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:34.926229522Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-04T12:04:39.28054777Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:04:39.442054756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:40.452519942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:04:43.635712152Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-04T12:04:44.442118054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:04:47.549386855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56850"
level=debug ts=2026-05-04T12:04:47.989991669Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-04T12:04:49.407702286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:04:49.417306201Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:04:49.423523199Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:04:49.424573147Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:04:49.442304459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:04:49.442340854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:04:49.442351792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:04:49.442361263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:04:52.345164515Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-04T12:04:54.442157724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:04:55.042302904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:04:56.699744116Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-04T12:04:58.601335525Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:04:59.441830123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:01.05527457Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-04T12:05:03.133117433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:05:03.143124376Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:05:04.441574408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:05.410143362Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-05-04T12:05:09.44195104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:09.764709253Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-04T12:05:14.119041893Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:05:14.44213932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:05:16.365029689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58352"
level=debug ts=2026-05-04T12:05:18.474007121Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=debug ts=2026-05-04T12:05:18.474043879Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=info ts=2026-05-04T12:05:19.441856948Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:05:19.441887571Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:05:19.441896472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:05:19.441889585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:22.828975642Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=warn ts=2026-05-04T12:05:24.44203913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:05:25.045575208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:05:27.183776967Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-04T12:05:29.441953375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:31.539071367Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-04T12:05:32.139824812Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:05:33.041952493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:05:34.441376038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:35.893289317Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-04T12:05:39.443792897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:40.248613079Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-04T12:05:41.305447046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:05:41.314273814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:05:44.441371381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:44.602833382Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=debug ts=2026-05-04T12:05:44.602864168Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
ts=2026-05-04T12:05:46.366500187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44524"
level=debug ts=2026-05-04T12:05:48.965763819Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-04T12:05:48.965787065Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=debug ts=2026-05-04T12:05:49.424120657Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:05:49.425148837Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:05:49.441546092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:05:49.441564847Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:05:49.441572857Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:05:49.4415953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:05:49.919721426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:05:49.928621879Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:05:53.313221805Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
ts=2026-05-04T12:05:53.847936031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:37624"
level=warn ts=2026-05-04T12:05:54.441225159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:05:55.048863235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:05:57.667653716Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-04T12:05:59.441619107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:05:59.516484786Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:06:02.02280778Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:06:04.44121645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:06:06.377249795Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-04T12:06:09.441390035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:06:10.732295957Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-04T12:06:14.441994114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:06:15.086840243Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-04T12:06:18.62413612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:06:19.442030406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:06:19.442042647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:06:19.442059141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-04T12:06:19.442049977Z caller=checkpoint.go:384 msg="flushing series" totalSize="282 kB" series=6
level=info ts=2026-05-04T12:06:19.442067739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:06:19.563718234Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-04T12:06:19.565290299Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.124089035s
level=warn ts=2026-05-04T12:06:24.441437907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:06:25.051003254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:06:27.860503417Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:06:29.441315852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:06:34.441983627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:06:39.442048404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:06:40.336096456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:06:40.344248586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:06:42.918832054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:06:42.927297321Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:06:44.441996518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:06:46.371875802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:40200"
ts=2026-05-04T12:06:47.558373043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38442"
level=debug ts=2026-05-04T12:06:49.423206447Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:06:49.42528445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:06:49.443253331Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:06:49.443287691Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:06:49.443299858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:06:49.44329156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:06:49.443436272Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:06:49.443558089Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-04T12:06:49.443568157Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:06:49.443604609Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.403µs success=true
level=debug ts=2026-05-04T12:06:49.443889865Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T12:06:52.950827752Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T12:06:54.441225391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:06:55.053366006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:06:56.457312009Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-04T12:06:59.441624854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:06:59.964366051Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-04T12:07:01.542204135Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:07:03.470522633Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:07:04.442212166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:06.976744557Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-04T12:07:09.441419184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:10.483449453Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-04T12:07:13.052771598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:07:13.054884394Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:07:13.990318037Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:07:14.442178395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:17.49620621Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T12:07:17.559656754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:46590"
level=info ts=2026-05-04T12:07:19.44120084Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:07:19.441216018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:19.441228923Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:07:19.441236963Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:07:21.002817683Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-04T12:07:24.441522945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:24.509847652Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:07:25.055611891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T12:07:25.113328052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:07:25.126228787Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T12:07:25.229582074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:07:25.237466185Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:07:25.237486099Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:07:25.237508079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:07:25.244596725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:07:28.015475478Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-04T12:07:29.442222049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:31.522904702Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=debug ts=2026-05-04T12:07:31.522934815Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=debug ts=2026-05-04T12:07:31.55795847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:07:34.442127497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:35.029115665Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-04T12:07:38.535217345Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-04T12:07:39.441241697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:42.042048833Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-04T12:07:42.605813871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:07:42.614954787Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:07:44.44126145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:45.552420785Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-04T12:07:49.054733487Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=debug ts=2026-05-04T12:07:49.423676765Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:07:49.424718381Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:07:49.441148503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:07:49.441170611Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:07:49.441180143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:07:49.441222033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:52.561128969Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-05-04T12:07:54.441222261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:07:55.056778299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:07:56.068268602Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-04T12:07:57.601635189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:07:59.441693455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:07:59.574493913Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=debug ts=2026-05-04T12:08:03.081207175Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T12:08:03.081234274Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=9
level=debug ts=2026-05-04T12:08:03.309116997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:08:04.44157863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:06.587261128Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-04T12:08:09.441497512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:10.093293509Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T12:08:13.421497828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:08:13.429723738Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:08:13.600269757Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-04T12:08:14.441903076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:08:16.893569461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:08:16.901687655Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:08:17.107246798Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-04T12:08:19.441906582Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:08:19.441934598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:08:19.441942146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:08:19.441941839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:20.612909145Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-04T12:08:24.119819072Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=warn ts=2026-05-04T12:08:24.441382999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:24.584911446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:08:25.058453192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:08:27.626242885Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-05-04T12:08:29.441693646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:31.135023809Z caller=checkpoint.go:372 msg="writing series" size="5.2 MB"
level=debug ts=2026-05-04T12:08:31.135044904Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=8
level=warn ts=2026-05-04T12:08:34.441530589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:34.639640243Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T12:08:38.14522456Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T12:08:39.44212105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:41.652246366Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:08:44.441293718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:45.159148236Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-04T12:08:47.564699269Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:51072"
level=debug ts=2026-05-04T12:08:48.375848238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:08:48.665609599Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-04T12:08:49.423637047Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:08:49.424684299Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:08:49.441141591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:08:49.441165893Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:08:49.441176118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:08:49.441250656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:08:50.682816967Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:08:52.171657674Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-04T12:08:54.441754374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:08:55.059304254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:08:55.678462177Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:08:59.184789079Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-04T12:08:59.441726939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:02.691014796Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-04T12:09:04.441897661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:06.197556023Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-04T12:09:07.157409915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:09:07.165494446Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:09:08.074180092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:09:08.082057929Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:09:09.441979582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:09.703886565Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-04T12:09:13.210333386Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=warn ts=2026-05-04T12:09:14.442005057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:09:16.390539567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48078"
level=debug ts=2026-05-04T12:09:16.717077489Z caller=checkpoint.go:372 msg="writing series" size="939 B"
ts=2026-05-04T12:09:17.565573082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37714"
level=debug ts=2026-05-04T12:09:17.927305762Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:09:19.4417619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:09:19.441763455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:19.441788691Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:09:19.441796274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:09:20.224160767Z caller=checkpoint.go:372 msg="writing series" size="424 kB"
level=debug ts=2026-05-04T12:09:20.2242141Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=14
level=debug ts=2026-05-04T12:09:23.729871139Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-04T12:09:24.44147532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:09:25.06154138Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:09:27.236315096Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T12:09:29.441354194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:30.743446834Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=debug ts=2026-05-04T12:09:34.250034714Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=warn ts=2026-05-04T12:09:34.441477426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:37.756347707Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-04T12:09:39.44148795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:41.262413305Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-04T12:09:44.441623441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:44.768786048Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=debug ts=2026-05-04T12:09:48.275131178Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-04T12:09:49.423180572Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:09:49.425290012Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:09:49.44174066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:09:49.441763793Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:09:49.441771882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:09:49.441797072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:09:49.976594257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:09:50.181422522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:09:50.192277224Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:09:50.225692996Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:09:51.782039138Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T12:09:51.951578399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:09:51.959280883Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:09:51.959303013Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:09:51.95932736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:09:51.966281427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:09:54.441595293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:09:55.064336579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:09:55.288730615Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-04T12:09:58.795256874Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=debug ts=2026-05-04T12:09:58.795286876Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-04T12:09:59.441799803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:02.301127993Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-04T12:10:04.441464894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:10:05.097781753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:10:05.109278768Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:10:05.808261646Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-04T12:10:09.315152615Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-04T12:10:09.441379439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:12.821014754Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=warn ts=2026-05-04T12:10:14.441404488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:16.328125126Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=debug ts=2026-05-04T12:10:16.328154885Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-05-04T12:10:17.567852996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:50364"
level=debug ts=2026-05-04T12:10:18.579128491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:10:19.441139461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:10:19.441175546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:10:19.441208316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:10:19.441220689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:19.834064185Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-04T12:10:23.341155766Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-04T12:10:24.441903592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:10:25.065863515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:10:26.847413902Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-04T12:10:29.441528721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:30.353603969Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-04T12:10:33.860448624Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-04T12:10:34.441725865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:10:35.042322152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:10:35.052828307Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:10:37.366917848Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-04T12:10:38.752789233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:10:39.442134192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:40.886817184Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-04T12:10:40.886845928Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=7
level=info ts=2026-05-04T12:10:41.874248835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:10:41.881987982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:10:44.379037169Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-04T12:10:44.441475163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:10:46.39796603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58544"
level=debug ts=2026-05-04T12:10:47.88567938Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-05-04T12:10:49.423075628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:10:49.425212024Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:10:49.441656055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:10:49.441671369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:10:49.441678353Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:10:49.441704543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:51.392257954Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-05-04T12:10:52.553120856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:10:53.880692123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:48046"
level=warn ts=2026-05-04T12:10:54.441540138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:10:54.89922196Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-04T12:10:55.06701376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:10:58.405894174Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=warn ts=2026-05-04T12:10:59.441418473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:11:01.912277708Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:11:04.442081356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:11:05.418899261Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-05-04T12:11:08.925448969Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-04T12:11:09.441404371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:11:12.431592599Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-04T12:11:14.441446592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:11:15.937869611Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
ts=2026-05-04T12:11:16.399652007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34430"
ts=2026-05-04T12:11:17.572598711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60482"
level=info ts=2026-05-04T12:11:19.44199898Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:11:19.442027691Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:11:19.442037267Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:11:19.442049782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:11:19.444158033Z caller=checkpoint.go:384 msg="flushing series" totalSize="987 kB" series=10
level=debug ts=2026-05-04T12:11:19.640809299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:11:19.688053178Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-04T12:11:19.692963987Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.249157001s
level=warn ts=2026-05-04T12:11:24.441592149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:11:25.067942388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:11:26.938926278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:11:26.940605816Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T12:11:29.441585674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:11:29.661215303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:11:29.670948262Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:11:34.442032009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:11:36.003967825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:11:36.011302484Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:11:39.442028144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:11:44.44215342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:11:46.401238899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46762"
ts=2026-05-04T12:11:47.57407284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60418"
level=debug ts=2026-05-04T12:11:49.424083445Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:11:49.425208692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:11:49.441617884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:11:49.441633821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:11:49.441627909Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:11:49.441640872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:11:49.441719151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:11:49.441753859Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-04T12:11:49.441761638Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:11:49.441777855Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.022µs success=true
level=debug ts=2026-05-04T12:11:49.441943612Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T12:11:51.609610066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:11:52.949390065Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T12:11:54.441557748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:11:55.069050134Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:11:56.455986638Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T12:11:59.442170843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:11:59.962214678Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-04T12:12:03.468135303Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:12:04.441759596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:06.975318728Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-04T12:12:09.441840751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:10.481975711Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:12:13.987997935Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:12:14.441792488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:17.494907815Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:12:19.441277141Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:12:19.44131658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:12:19.441325793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:12:19.441341769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:12:20.749595858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:12:20.76061919Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:12:21.002099474Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-04T12:12:24.44172439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:24.507505136Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:12:25.071714941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:12:27.262289821Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:12:28.013469214Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-04T12:12:28.380746384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:12:29.441622471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:12:30.643829095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:12:30.651059824Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:12:31.520865725Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
level=debug ts=2026-05-04T12:12:31.520886591Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=13
level=warn ts=2026-05-04T12:12:34.44211299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:35.02732871Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=debug ts=2026-05-04T12:12:38.533526921Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T12:12:39.441281084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:42.039576656Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-05-04T12:12:44.441847217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:45.546945585Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=debug ts=2026-05-04T12:12:49.052956279Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-04T12:12:49.424281536Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:12:49.424602368Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:12:49.442160616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:12:49.442182171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:12:49.442204544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:12:49.442241153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:52.559624898Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=warn ts=2026-05-04T12:12:54.442067621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:12:55.073961039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:12:56.065338536Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-04T12:12:57.780676251Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:12:59.441806079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:12:59.57285558Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-04T12:13:01.173494597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:13:01.187393662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:13:03.080117154Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-04T12:13:03.080139973Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=9
level=warn ts=2026-05-04T12:13:04.4412576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:06.585120798Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-04T12:13:09.441262941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:10.092362475Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T12:13:13.59960318Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T12:13:13.599625426Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-04T12:13:14.442156745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:17.104506722Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-04T12:13:17.578524847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:41296"
level=debug ts=2026-05-04T12:13:17.772176252Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:13:19.441161016Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:13:19.441202899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:13:19.441211247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:13:19.441228527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:13:20.082897998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:13:20.091798487Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:13:20.611336792Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-04T12:13:24.118392263Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-04T12:13:24.441954062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:13:25.075653931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:13:25.888358644Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:13:27.624319791Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-05-04T12:13:29.441441891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:31.133902567Z caller=checkpoint.go:372 msg="writing series" size="5.7 MB"
level=debug ts=2026-05-04T12:13:31.133923814Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.3 MB" series=5
level=warn ts=2026-05-04T12:13:34.441394321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:34.6371807Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T12:13:38.144119304Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T12:13:39.442164213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:41.650404992Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:13:44.441865582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:45.157272731Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-04T12:13:48.663823013Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-04T12:13:49.424275749Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:13:49.424575483Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:13:49.441971228Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:13:49.441989273Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:13:49.441996827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:13:49.442043752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:13:52.170503277Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=warn ts=2026-05-04T12:13:54.442225866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:13:55.077042957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:13:55.119006928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:13:55.675992975Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-04T12:13:58.213689543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:13:58.221592789Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:13:59.18237808Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-04T12:13:59.442154195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:01.309447579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:14:02.689630333Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T12:14:04.25807053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:14:04.266665646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:14:04.2666844Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:14:04.266705274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:14:04.272902897Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:14:04.441220205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:06.195329844Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-04T12:14:06.19535183Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-04T12:14:09.44143276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:09.702533846Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-04T12:14:13.209265346Z caller=checkpoint.go:372 msg="writing series" size="850 kB"
level=debug ts=2026-05-04T12:14:13.209282003Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-04T12:14:14.441213712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:16.715654596Z caller=checkpoint.go:372 msg="writing series" size="939 B"
ts=2026-05-04T12:14:17.582482193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44078"
level=info ts=2026-05-04T12:14:19.441859042Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:14:19.441891636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:14:19.4418999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:14:19.441952073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:20.222253632Z caller=checkpoint.go:372 msg="writing series" size="861 kB"
level=info ts=2026-05-04T12:14:22.471441264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:14:22.479868458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:14:23.728610321Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-04T12:14:24.441815587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:14:25.078539173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:14:25.361631372Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:14:27.234845935Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T12:14:29.441931172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:30.741003052Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-04T12:14:34.247815648Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-04T12:14:34.441222409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:37.753812072Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:14:37.753839964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-04T12:14:38.926275175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:14:39.442047794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:14:40.952266791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:14:40.960143129Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:14:41.261042384Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:14:44.44190717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:14:44.766985108Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
ts=2026-05-04T12:14:46.421476725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34494"
level=debug ts=2026-05-04T12:14:48.273486706Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-04T12:14:49.423503854Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:14:49.424551432Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:14:49.442036007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:14:49.442064787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:14:49.442053798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:14:49.442073149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:14:51.780106453Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-04T12:14:53.389125549Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:14:54.442046664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:14:55.079818423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:14:55.286578679Z caller=checkpoint.go:372 msg="writing series" size="443 kB"
level=info ts=2026-05-04T12:14:55.993304348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:14:56.001215914Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:14:58.793538506Z caller=checkpoint.go:372 msg="writing series" size="658 kB"
level=debug ts=2026-05-04T12:14:58.793562686Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=warn ts=2026-05-04T12:14:59.44158153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:02.299774082Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-04T12:15:04.441914763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:05.806247781Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-04T12:15:09.313221945Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=warn ts=2026-05-04T12:15:09.441901285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:12.819035601Z caller=checkpoint.go:372 msg="writing series" size="492 kB"
level=warn ts=2026-05-04T12:15:14.442113777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:16.325410721Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=debug ts=2026-05-04T12:15:16.325437179Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
ts=2026-05-04T12:15:16.422585562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45910"
ts=2026-05-04T12:15:17.585085262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:55584"
level=info ts=2026-05-04T12:15:19.441695839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:15:19.441729063Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:15:19.441737877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:15:19.44177229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:19.832560391Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-04T12:15:23.338623113Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=warn ts=2026-05-04T12:15:24.442039406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:15:25.081651116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:15:26.845263561Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-04T12:15:28.927202966Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:15:28.927250099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:15:28.936348841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:15:29.441432099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:30.351811931Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-04T12:15:33.858011176Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=warn ts=2026-05-04T12:15:34.441542338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:37.364984107Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-04T12:15:37.576684829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:15:37.578346607Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T12:15:39.39565904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:15:39.406547461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:15:39.441785677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:40.901818709Z caller=checkpoint.go:372 msg="writing series" size="28 MB"
level=debug ts=2026-05-04T12:15:40.901843279Z caller=checkpoint.go:384 msg="flushing series" totalSize="29 MB" series=7
level=debug ts=2026-05-04T12:15:44.377304708Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-04T12:15:44.441407682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:15:45.83367104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:15:45.842557651Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:15:46.3198347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:15:46.327797656Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:15:47.88427596Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-04T12:15:49.422945536Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:15:49.425059991Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:15:49.425280258Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T12:15:49.425317347Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975440 n=1
level=debug ts=2026-05-04T12:15:49.425357514Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975440 n=1
level=info ts=2026-05-04T12:15:49.441401486Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:15:49.441418344Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:15:49.441426345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:15:49.441443947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:51.390610279Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-05-04T12:15:53.922440342Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:57274"
level=warn ts=2026-05-04T12:15:54.442218215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:15:54.896646087Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
ts=2026-05-04T12:15:55.083555892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:15:58.40378475Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=warn ts=2026-05-04T12:15:59.441667524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:16:00.791311524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:16:01.909712485Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:16:04.442024359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:16:05.416657746Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=debug ts=2026-05-04T12:16:08.922924278Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-04T12:16:09.44124623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:16:10.790691856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:16:10.798476469Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:16:12.429690104Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-04T12:16:14.441605787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:16:15.936815284Z caller=checkpoint.go:372 msg="writing series" size="625 kB"
level=debug ts=2026-05-04T12:16:15.936837924Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-04T12:16:19.442049838Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:16:19.442081528Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:16:19.44209055Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:16:19.442092406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:16:19.44605577Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-04T12:16:19.447492578Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00561152s
level=debug ts=2026-05-04T12:16:21.932068781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:16:24.442055106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:16:25.085273547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:16:28.17566977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:16:28.183047689Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:16:29.441450478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:16:31.167105124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:16:31.87459044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:16:31.882387659Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:16:31.882414713Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:16:31.882448092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:16:31.890209449Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:16:34.442169877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:16:39.441265096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:16:44.441311703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:16:48.263669741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:16:48.274714783Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-04T12:16:49.424321039Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:16:49.424391205Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:16:49.441070151Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:16:49.441249735Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-04T12:16:49.441260962Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:16:49.441286667Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.946µs success=true
level=warn ts=2026-05-04T12:16:49.441344306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:16:49.441360627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:16:49.441380368Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:16:49.441390666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:16:49.441534383Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T12:16:52.903687809Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
ts=2026-05-04T12:16:53.929087498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52018"
level=warn ts=2026-05-04T12:16:54.441536311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:16:55.086430288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:16:56.365080371Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T12:16:59.44124012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:16:59.630515542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:16:59.827148806Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-04T12:17:02.093026138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:17:02.102857122Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:17:03.288474703Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:17:04.4414651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:06.750370075Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-04T12:17:07.13381602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:17:09.441364698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:10.211323358Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:17:13.672480347Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:17:14.441393989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:17:15.358155333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:17:15.36572641Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:17:17.134762171Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:17:19.4415909Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:17:19.441617391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:17:19.441625264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:17:19.441668068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:20.595916227Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-04T12:17:24.057736865Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=warn ts=2026-05-04T12:17:24.441519415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:17:25.087551159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:17:27.519010208Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:17:27.519042286Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-05-04T12:17:27.969462606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:17:27.979093024Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:17:28.403567611Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:17:28.403639621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:17:28.417231251Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:17:28.703847411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:17:28.713205398Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:17:29.441595005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:30.98065851Z caller=checkpoint.go:372 msg="writing series" size="852 kB"
level=warn ts=2026-05-04T12:17:34.441987353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:34.442010269Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-04T12:17:37.908314777Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T12:17:39.441881033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:41.365262297Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-04T12:17:41.365285157Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T12:17:41.383376759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:17:41.398997839Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:17:44.441953487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:44.82743634Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=debug ts=2026-05-04T12:17:46.627424396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:17:47.20132337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:17:47.206090965Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T12:17:47.597511053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:57826"
level=debug ts=2026-05-04T12:17:48.288405926Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-04T12:17:49.423700251Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:17:49.42477425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:17:49.441151111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:17:49.441170733Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:17:49.441178419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:17:49.441215223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:17:51.75009333Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-04T12:17:54.441757732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:17:55.089626715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:17:55.211591707Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-04T12:17:57.358918978Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:17:58.672533022Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-04T12:17:59.374945815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:17:59.382672228Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:17:59.442256385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:02.135630537Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-04T12:18:02.13565513Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
level=warn ts=2026-05-04T12:18:04.441420289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:18:04.643874254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:18:04.654924368Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:18:05.595438997Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T12:18:07.42305852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:18:07.430161997Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:18:09.057274439Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:18:09.441793158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:12.522026325Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-04T12:18:12.522056567Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=warn ts=2026-05-04T12:18:14.441535672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:15.980702367Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-04T12:18:16.678228424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:18:16.683489647Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:18:19.441288021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:18:19.441318316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:18:19.441336237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:18:19.441342621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:19.44255799Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-04T12:18:22.541310233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:18:22.904313567Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
ts=2026-05-04T12:18:23.939031066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:53864"
level=warn ts=2026-05-04T12:18:24.442045397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:18:25.091944082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:18:26.3655075Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-04T12:18:26.764827693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:18:26.772789386Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:18:29.418386417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:18:29.441293905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:29.830675724Z caller=checkpoint.go:372 msg="writing series" size="6.4 MB"
level=debug ts=2026-05-04T12:18:29.830697443Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.0 MB" series=5
level=debug ts=2026-05-04T12:18:33.288028731Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-04T12:18:34.442136488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:36.749273365Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-04T12:18:39.441517744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:40.210793467Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-04T12:18:43.673125293Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-04T12:18:44.441548331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:18:46.437933228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55778"
level=debug ts=2026-05-04T12:18:47.134231626Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-04T12:18:49.424489811Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:18:49.424494614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:18:49.441998565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:18:49.442016515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:18:49.442027157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:18:49.442041538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:18:50.595540114Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-04T12:18:51.689816213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:18:51.69900467Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:18:52.053159332Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:18:54.057359398Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=warn ts=2026-05-04T12:18:54.442002702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:18:55.093165094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:18:57.519520814Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-04T12:18:58.653846343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:18:58.660483808Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:18:59.44210206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:18:59.805881806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:18:59.810545657Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:19:00.980354835Z caller=checkpoint.go:372 msg="writing series" size="399 kB"
level=debug ts=2026-05-04T12:19:00.980376644Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-05-04T12:19:03.248820364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:19:03.256212778Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:19:04.441500221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:04.441573547Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T12:19:05.087258599Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:19:05.089078451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:19:07.903641586Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-05-04T12:19:09.442134503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:11.36579137Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=warn ts=2026-05-04T12:19:14.441916359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:19:14.504268659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:19:14.514103753Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:19:14.827127796Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-04T12:19:14.827150372Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=debug ts=2026-05-04T12:19:18.287712751Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T12:19:19.441936518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:19:19.441963021Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:19:19.441973412Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:19:19.441983872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:21.750296309Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T12:19:21.750318171Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-04T12:19:24.441379784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:19:25.09551174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:19:25.211407291Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-04T12:19:26.012726143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:19:28.672999597Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=warn ts=2026-05-04T12:19:29.442009196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:32.134615397Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-04T12:19:34.441849227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:35.595701435Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=debug ts=2026-05-04T12:19:39.05749283Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-04T12:19:39.441880944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:19:41.754565218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:19:41.764388613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:19:42.518872676Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-04T12:19:44.442051535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:45.981021634Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-04T12:19:49.42342564Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:19:49.424468373Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:19:49.441983309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:19:49.442005868Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:19:49.442014542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:19:49.442054453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:49.442090126Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-04T12:19:52.904038663Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
ts=2026-05-04T12:19:53.949048821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56850"
level=warn ts=2026-05-04T12:19:54.441833801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:19:54.522839532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:19:54.52786152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T12:19:55.09773312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:19:55.381923712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:19:55.388903461Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:19:55.929321674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:19:56.3656867Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=debug ts=2026-05-04T12:19:56.598938915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:19:57.977251618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:19:57.983502369Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:19:59.441722017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:19:59.827098842Z caller=checkpoint.go:372 msg="writing series" size="835 kB"
level=debug ts=2026-05-04T12:19:59.827122729Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=11
level=debug ts=2026-05-04T12:20:03.288485614Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=warn ts=2026-05-04T12:20:04.441374898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:06.749478178Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:20:09.441221524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:10.211478915Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-04T12:20:13.673347453Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=info ts=2026-05-04T12:20:14.081320264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:20:14.09308795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-04T12:20:14.441250377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:17.134596245Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-04T12:20:17.134617523Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-05-04T12:20:17.617167464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:57796"
level=info ts=2026-05-04T12:20:19.441283622Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:20:19.441307537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:20:19.44131525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:20:19.441336949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:20.595942078Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-04T12:20:24.059392202Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-04T12:20:24.441890493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:20:25.099076031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:20:27.519095298Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-04T12:20:29.13077598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:20:29.442009419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:20:29.920722129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:20:29.929234471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:20:29.929261669Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:20:29.92929332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:20:29.936701791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:20:30.980530711Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T12:20:34.442145084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:34.442218715Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-05-04T12:20:37.903973711Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-04T12:20:39.441200738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:20:40.756830776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:20:40.76559354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:20:41.399589106Z caller=checkpoint.go:372 msg="writing series" size="37 MB"
level=debug ts=2026-05-04T12:20:41.399615796Z caller=checkpoint.go:384 msg="flushing series" totalSize="37 MB" series=7
level=info ts=2026-05-04T12:20:41.698830478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:20:41.703409832Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:20:44.226461505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:20:44.234279403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:20:44.441913709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:44.826976781Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-04T12:20:47.714882952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:20:48.288631049Z caller=checkpoint.go:372 msg="writing series" size="701 kB"
level=debug ts=2026-05-04T12:20:49.42336219Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:20:49.424407037Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:20:49.441921424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:20:49.441939639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:20:49.441947665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:20:49.44195759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:20:51.749459908Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-04T12:20:54.441593988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:20:55.10045446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:20:55.211209453Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=debug ts=2026-05-04T12:20:55.211229489Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T12:20:55.38865815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:20:55.399581483Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:20:58.672683206Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:20:59.445233548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:20:59.882453982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:20:59.89016085Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:21:00.760695055Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:21:02.134781893Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:21:04.441363569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:05.596227518Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-05-04T12:21:09.057347484Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=info ts=2026-05-04T12:21:09.213647305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:21:09.222118206Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:21:09.441881837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:12.519502289Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-04T12:21:14.441608388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:15.98085649Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:21:15.980880475Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-04T12:21:16.449758369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36948"
level=info ts=2026-05-04T12:21:19.441245975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:21:19.441277805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:21:19.441288114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:21:19.441317022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:21:19.444018712Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-04T12:21:19.447293299Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005834727s
level=warn ts=2026-05-04T12:21:24.441564763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:25.018709527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:21:25.10212825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:21:29.030360234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:21:29.441697745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:21:31.571471357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:21:31.575476279Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:21:34.441790105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:21:37.574232357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:21:37.582868456Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:21:39.120878015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:21:39.127574641Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:21:39.441679626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:21:44.441723286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:49.423340581Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:21:49.424378972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:21:49.440828229Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:21:49.441126153Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-04T12:21:49.441136252Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:21:49.441225944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T12:21:49.44123151Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.311µs success=true
level=debug ts=2026-05-04T12:21:49.441246235Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:21:49.441256273Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:21:49.441257198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:49.441445501Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T12:21:52.860035326Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
ts=2026-05-04T12:21:53.962557428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:41122"
level=warn ts=2026-05-04T12:21:54.441860635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:21:55.104864033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:21:56.276912612Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T12:21:56.491686255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:21:56.49935544Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:21:58.833139424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:21:59.441865577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:21:59.694710336Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-04T12:22:03.113202071Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T12:22:03.545211921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:22:03.55327975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:22:04.442103835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:22:06.531246217Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=warn ts=2026-05-04T12:22:09.441281799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:22:09.65297944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:22:09.947961162Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:22:13.36553017Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:22:14.441511689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:22:16.454143153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:50660"
level=debug ts=2026-05-04T12:22:16.783438762Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:22:19.44215843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:22:19.442204871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:22:19.442212933Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:22:19.442203433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:22:20.040626405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:22:20.046216419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:22:20.201177408Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-04T12:22:23.619455339Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=warn ts=2026-05-04T12:22:24.441224236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:22:25.107752055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:22:25.799254259Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:22:27.037372759Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:22:27.037390226Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=info ts=2026-05-04T12:22:27.988617733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:22:27.997849058Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:22:29.442153194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:22:30.455165247Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-04T12:22:33.87252481Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-04T12:22:33.872547542Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-04T12:22:34.441808759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:22:35.470449987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:22:35.477031183Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:22:35.477051683Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:22:35.477074863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:22:35.482772948Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:22:37.289614588Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-04T12:22:39.44208703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:22:40.707877803Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-05-04T12:22:44.12570059Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=warn ts=2026-05-04T12:22:44.442227943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:22:45.99080915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:22:46.000498268Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:22:47.543054549Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
ts=2026-05-04T12:22:47.631016982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:48422"
level=debug ts=2026-05-04T12:22:49.423431438Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:22:49.424484119Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:22:49.441941756Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:22:49.441959426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:22:49.441967587Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:22:49.441969412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:22:50.961203963Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=debug ts=2026-05-04T12:22:53.810173974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:22:53.968598044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:40572"
level=info ts=2026-05-04T12:22:54.035616222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:22:54.043240427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:22:54.379093512Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:22:54.441304598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:22:55.109423975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T12:22:56.371318685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="309 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:22:56.379123124Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-04T12:22:57.086302688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:22:57.087806215Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:22:57.796718533Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-04T12:22:59.441800828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:23:00.568240452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:23:00.572900183Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:23:01.216896214Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-04T12:23:01.216918379Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=8
level=warn ts=2026-05-04T12:23:04.441219888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:04.632253974Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T12:23:08.049960583Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:23:09.441256543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:11.472583989Z caller=checkpoint.go:372 msg="writing series" size="4.7 MB"
level=debug ts=2026-05-04T12:23:11.472605062Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=3
level=warn ts=2026-05-04T12:23:14.442026955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:14.885113775Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-04T12:23:15.354763835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:23:15.365850291Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:23:16.390472911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:23:16.397236957Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:23:18.30259462Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-04T12:23:19.441203303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:23:19.441239573Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:23:19.441250422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:23:19.441249975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:20.091988337Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:23:21.720553889Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=warn ts=2026-05-04T12:23:24.441589428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:23:25.111263509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:23:25.138584252Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-04T12:23:28.56284156Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-05-04T12:23:28.562865516Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.9 MB" series=5
level=warn ts=2026-05-04T12:23:29.441232784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:31.973773102Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-04T12:23:32.536314947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:23:34.442131985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:35.391798799Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T12:23:38.809396148Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-04T12:23:38.852631698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:23:38.859116396Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:23:39.441383611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:42.227202697Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:23:44.442129273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:23:45.644657288Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-04T12:23:47.344089295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:23:49.06223276Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-04T12:23:49.424032406Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:23:49.425100233Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:23:49.441489047Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:23:49.441506552Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:23:49.441499674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:23:49.44151455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:23:52.480364902Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-04T12:23:54.442047163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:23:54.717446161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="309 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:23:54.724749276Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df2d59797%3A19df2deb893%3A6fdb862b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-04T12:23:54.724767521Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:23:54.724788599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:23:54.729558332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:23:55.032226786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:23:55.036764716Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:23:55.036784196Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:23:55.036805613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:23:55.040298283Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T12:23:55.11265104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:23:55.898207701Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=debug ts=2026-05-04T12:23:59.315679255Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:23:59.315703218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-04T12:23:59.396828047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:23:59.405326018Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:23:59.40535184Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:23:59.405377762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:23:59.412389198Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:23:59.441228817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:02.733805223Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=warn ts=2026-05-04T12:24:04.441894307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:24:05.429876201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:24:05.435866447Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:24:06.150616965Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T12:24:07.908813872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:24:07.915762087Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:24:09.283552998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:24:09.289327755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:24:09.441228146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:09.5683443Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-04T12:24:12.151468672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:24:12.159488713Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:24:12.986511926Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=warn ts=2026-05-04T12:24:14.441994286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:16.405546336Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-04T12:24:16.405569441Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=warn ts=2026-05-04T12:24:19.441577941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:24:19.441577899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:24:19.441614941Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:24:19.441625761Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:24:19.821954833Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-04T12:24:22.051466965Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:24:23.240987837Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-04T12:24:23.241010228Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
ts=2026-05-04T12:24:23.977878712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:43418"
level=warn ts=2026-05-04T12:24:24.441996633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:24:25.115683998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:24:26.244454901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:24:26.25148111Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:24:26.657634155Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-04T12:24:27.026816413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:24:27.034635705Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:24:29.441847293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:30.075043803Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=debug ts=2026-05-04T12:24:33.492938294Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-04T12:24:34.177449182Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:24:34.441235208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:36.910609716Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-04T12:24:39.441757357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:40.327677101Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-04T12:24:43.423800473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:24:43.435182379Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:24:43.745981528Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-04T12:24:44.441434519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:24:45.141832024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:24:45.147695568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:24:46.458382453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:24:46.465546649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44310"
level=debug ts=2026-05-04T12:24:47.163912258Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
ts=2026-05-04T12:24:47.639233569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38994"
level=debug ts=2026-05-04T12:24:49.422884229Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:24:49.4250107Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:24:49.441360443Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:24:49.441379167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:24:49.441389272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:24:49.441400609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:24:50.581479737Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-04T12:24:53.999071628Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-04T12:24:54.441609446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:24:55.118418446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:24:57.41733305Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-04T12:24:57.417358238Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-05-04T12:24:59.441321623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:00.834850408Z caller=checkpoint.go:372 msg="writing series" size="888 kB"
level=info ts=2026-05-04T12:25:03.177033908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:25:03.186823449Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:25:04.25212189Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-04T12:25:04.442104736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:07.670325649Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:25:09.441958923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:11.087966054Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=debug ts=2026-05-04T12:25:11.087987648Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-04T12:25:14.441437326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:14.505375558Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=debug ts=2026-05-04T12:25:15.375489792Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:25:17.923057515Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=debug ts=2026-05-04T12:25:17.923079812Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-04T12:25:19.441775975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:25:19.441808038Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:25:19.441810935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:25:19.441818014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:25:21.341134817Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-04T12:25:22.442078956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:25:22.447445804Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:25:24.247786622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:25:24.255987333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:25:24.441249674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:24.758306649Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
ts=2026-05-04T12:25:25.119405283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:25:25.223534864Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:25:28.176674873Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-04T12:25:29.441894399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:31.594271032Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T12:25:34.010744053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:25:34.016780715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:25:34.442096385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:35.012038458Z caller=checkpoint.go:372 msg="writing series" size="382 kB"
level=info ts=2026-05-04T12:25:36.895132343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:25:36.903798649Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:25:38.429450405Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:25:39.441919264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:41.883737954Z caller=checkpoint.go:372 msg="writing series" size="41 MB"
level=debug ts=2026-05-04T12:25:41.883772954Z caller=checkpoint.go:384 msg="flushing series" totalSize="42 MB" series=7
level=warn ts=2026-05-04T12:25:44.441695498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:45.265225879Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
ts=2026-05-04T12:25:47.641416927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38508"
level=debug ts=2026-05-04T12:25:48.682451639Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=info ts=2026-05-04T12:25:49.424400599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:25:49.424409912Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:25:49.441900738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:25:49.441917471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:25:49.44192466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:25:49.441929339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:25:50.387456098Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:25:52.100310474Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-04T12:25:52.232895667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:25:52.23954501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:25:54.441568496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:25:55.121716586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:25:55.5179503Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-04T12:25:55.517972241Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-04T12:25:58.93587689Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:25:59.442214877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:26:02.353833737Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:26:04.441752025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:26:05.771022686Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=debug ts=2026-05-04T12:26:09.189252817Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-04T12:26:09.441260873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:26:12.606319853Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-04T12:26:12.694671415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:26:12.699709363Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:26:14.441939649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:26:14.777773745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:26:14.785550233Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:26:16.024126846Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:26:16.024148678Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-04T12:26:19.441274029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:26:19.441303701Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:26:19.441311616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:26:19.44135983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:26:19.596517714Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-04T12:26:19.598099445Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.156714085s
level=debug ts=2026-05-04T12:26:20.274731125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:26:20.704071744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:26:20.710180795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:26:23.51210076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:26:23.989300781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46408"
level=info ts=2026-05-04T12:26:24.260932277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:26:24.269771871Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:26:24.441956061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:26:25.12538522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=warn ts=2026-05-04T12:26:29.441349981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:26:34.441989721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:26:39.441894618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:26:44.441268026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:26:48.789617588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:26:48.797491579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:26:49.014529273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:26:49.423617036Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:26:49.424672338Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:26:49.441102504Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:26:49.441146977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:26:49.44116569Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:26:49.441173277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:26:49.441205994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:26:49.441237682Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-04T12:26:49.441247853Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:26:49.441271187Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.3µs success=true
level=debug ts=2026-05-04T12:26:49.441473037Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T12:26:52.85946902Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
ts=2026-05-04T12:26:53.992022716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42534"
level=warn ts=2026-05-04T12:26:54.441327408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:26:54.995691048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:26:55.003876247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:26:55.003897492Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:26:55.003923692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:26:55.010237049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T12:26:55.128001618Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:26:56.277752577Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T12:26:59.441783786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:26:59.695378046Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-04T12:27:03.113255316Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:27:04.441428224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:27:05.117384246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:27:05.122914331Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:27:05.367459157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:27:05.371898588Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:27:06.530271943Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-04T12:27:09.44195428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:27:09.948567576Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:27:13.365980641Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T12:27:13.426359032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:27:13.433619341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:27:14.442087379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:27:16.474418007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43052"
level=debug ts=2026-05-04T12:27:16.783495653Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:27:19.441393121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:27:19.441398614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:27:19.441421264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:27:19.441428996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:27:20.201434432Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-04T12:27:20.278970582Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:27:20.281014563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:27:21.2789476Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:27:22.60852749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:27:22.617778763Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:27:23.619446681Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:27:23.995804591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:37612"
level=warn ts=2026-05-04T12:27:24.441616295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:27:25.130470573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:27:27.037330018Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:27:27.037354922Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-05-04T12:27:29.441281165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:27:30.455086584Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-04T12:27:30.45511199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-04T12:27:33.872443615Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-04T12:27:34.441827935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:27:34.851552885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:27:34.860343006Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:27:37.290408597Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-04T12:27:39.441672072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:27:40.707417698Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-04T12:27:44.125513655Z caller=checkpoint.go:372 msg="writing series" size="738 kB"
level=warn ts=2026-05-04T12:27:44.441982538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:27:47.543877989Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-04T12:27:47.543899565Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-04T12:27:49.424005871Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:27:49.425073376Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:27:49.441470686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:27:49.4414928Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:27:49.441504248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:27:49.441500162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:27:50.961270193Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-04T12:27:51.669246683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:27:51.677312315Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:27:51.699351024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:27:51.703212318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:27:54.378333195Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:27:54.441474801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:27:55.132371199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:27:56.859239692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:27:56.859318806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:27:56.866637178Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:27:57.796336941Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-04T12:27:59.44214389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:01.216876873Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-04T12:28:01.216899185Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
level=warn ts=2026-05-04T12:28:04.441994296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:04.631631659Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T12:28:04.679796873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:28:04.686111533Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:28:05.612803613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:28:05.621306595Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:28:08.049562179Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:28:09.441640416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:11.470984302Z caller=checkpoint.go:372 msg="writing series" size="7.6 MB"
level=debug ts=2026-05-04T12:28:11.471007066Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.6 MB" series=3
level=warn ts=2026-05-04T12:28:14.441620179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:14.884809694Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-04T12:28:15.57984179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:28:15.586692736Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:28:18.302325298Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=debug ts=2026-05-04T12:28:19.302491825Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:28:19.442062323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:28:19.442092871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:28:19.442100982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:28:19.442150835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:21.721091555Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=info ts=2026-05-04T12:28:23.312059082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:28:23.320436614Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-04T12:28:24.003093478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46820"
level=warn ts=2026-05-04T12:28:24.441434176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:28:25.13343731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:28:25.138370672Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=debug ts=2026-05-04T12:28:26.833998354Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:28:28.559837488Z caller=checkpoint.go:372 msg="writing series" size="8.1 MB"
level=debug ts=2026-05-04T12:28:28.559859548Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.8 MB" series=5
level=warn ts=2026-05-04T12:28:29.441265578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:31.973700439Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-04T12:28:34.441496336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:35.391642976Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T12:28:36.614133282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:28:36.620906313Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:28:38.809702777Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T12:28:39.441996967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:28:42.22646126Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:28:44.441391114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:28:45.309743511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:28:45.318503597Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:28:45.644651196Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-04T12:28:46.150044521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:28:46.15616636Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-04T12:28:46.48191557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:55132"
level=info ts=2026-05-04T12:28:46.824297531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:28:46.832547814Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:28:49.062119936Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-04T12:28:49.423079168Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:28:49.425211074Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:28:49.441630564Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:28:49.441648668Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:28:49.44165715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:28:49.44165941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:28:50.603754062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:28:50.611009336Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:28:52.479743015Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-04T12:28:54.44137026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:28:55.135566727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:28:55.473524397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:28:55.897883447Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=debug ts=2026-05-04T12:28:59.314933676Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:28:59.31495299Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-04T12:28:59.441728268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:02.733335382Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=warn ts=2026-05-04T12:29:04.441299385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:06.150994287Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-04T12:29:09.441231199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:09.5689402Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-04T12:29:11.118156319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:29:12.986898693Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=debug ts=2026-05-04T12:29:12.986922811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-04T12:29:14.441682944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:16.405289168Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T12:29:16.405316601Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-04T12:29:19.331576094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:29:19.338935302Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:29:19.338954736Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:29:19.338974673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:29:19.344426884Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:29:19.441671235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:29:19.441681354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:19.441693282Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:29:19.441701666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:29:19.821997795Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-04T12:29:21.157813752Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:29:21.664530011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:29:21.668947253Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:29:23.240086467Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-04T12:29:23.240108264Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
ts=2026-05-04T12:29:24.010609201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:37612"
level=warn ts=2026-05-04T12:29:24.442170804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:29:25.138508576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:29:26.657610295Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-04T12:29:27.914117229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:29:27.921818154Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:29:27.991894461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:29:27.99777053Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:29:29.442102997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:30.074556429Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=info ts=2026-05-04T12:29:31.256935187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:29:31.264130963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:29:33.493126398Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=warn ts=2026-05-04T12:29:34.441720935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:29:36.502464137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:29:36.508941473Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:29:36.910071051Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-04T12:29:39.441702717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:40.327800537Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-04T12:29:42.278383176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:29:42.286612573Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:29:43.745567156Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-04T12:29:44.441622398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:29:46.486147279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45500"
level=debug ts=2026-05-04T12:29:47.163804216Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-04T12:29:49.423351509Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:29:49.42438712Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:29:49.44196631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:29:49.441994895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:29:49.442005073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:29:49.441969182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:29:50.58124525Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-04T12:29:51.374302771Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:29:53.999278311Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T12:29:53.999300316Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=9
ts=2026-05-04T12:29:54.01328561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56180"
level=warn ts=2026-05-04T12:29:54.44191151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:29:55.140239981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:29:57.417453837Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=warn ts=2026-05-04T12:29:59.441705948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:00.834982799Z caller=checkpoint.go:372 msg="writing series" size="890 kB"
level=debug ts=2026-05-04T12:30:00.835004901Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-04T12:30:00.934713341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:30:00.942838567Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:30:04.252648708Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-04T12:30:04.442021582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:07.669333049Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:30:09.441261701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:10.189038595Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:30:11.087944849Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=warn ts=2026-05-04T12:30:14.441742953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:14.505845106Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=info ts=2026-05-04T12:30:16.073028978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:30:16.081708215Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:30:16.081729959Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:30:16.081753321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:30:16.088602192Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T12:30:17.660061693Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:50478"
level=debug ts=2026-05-04T12:30:17.923550045Z caller=checkpoint.go:372 msg="writing series" size="598 kB"
level=debug ts=2026-05-04T12:30:17.923575353Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-04T12:30:19.156836956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:30:19.162534847Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:30:19.162553454Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:30:19.162573929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:30:19.167456596Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:30:19.442115909Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:30:19.442142037Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:30:19.442150074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:30:19.442145503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:21.126871464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:30:21.34051537Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-04T12:30:21.433010744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:30:21.440662108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:30:21.598919388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:30:21.602968451Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:30:21.60298607Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:30:21.603006775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:30:21.606455657Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:30:24.442027872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:24.758295024Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-04T12:30:25.141534315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:30:28.176294082Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-04T12:30:29.442130407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:30:31.009316226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:30:31.018040142Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:30:31.593413147Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T12:30:32.446036204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:30:32.451825068Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:30:32.862809826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:30:32.866845569Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:30:34.441351829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:35.011486867Z caller=checkpoint.go:372 msg="writing series" size="311 kB"
level=info ts=2026-05-04T12:30:35.12093577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:30:35.126102985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:30:38.430005486Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:30:39.441682414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:41.883667783Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-04T12:30:41.88369605Z caller=checkpoint.go:384 msg="flushing series" totalSize="46 MB" series=7
level=warn ts=2026-05-04T12:30:44.441613748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:45.264506146Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-04T12:30:48.682843997Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=info ts=2026-05-04T12:30:49.424586128Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:30:49.424606263Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=debug ts=2026-05-04T12:30:49.425898853Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T12:30:49.425943071Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975441 n=1
level=debug ts=2026-05-04T12:30:49.425990539Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975441 n=1
level=info ts=2026-05-04T12:30:49.442115988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:30:49.442139504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:30:49.442148526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:30:49.442163971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:30:49.86697452Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:30:52.100062911Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-05-04T12:30:54.019957216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52954"
level=warn ts=2026-05-04T12:30:54.441458065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:30:55.143445087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:30:55.518215175Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T12:30:55.5182381Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T12:30:57.783746219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:30:57.793884878Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:30:58.93628491Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:30:59.441825475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:31:01.569316057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:31:01.577957912Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:31:02.353129274Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=info ts=2026-05-04T12:31:03.855259522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:31:03.871714117Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:31:04.44195301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:31:05.770694737Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-04T12:31:09.026227585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:31:09.030336789Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:31:09.188704793Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-04T12:31:09.441713133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:31:10.985942092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:31:12.606775884Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-04T12:31:13.678125645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:31:13.68509153Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:31:14.441264364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:31:14.973248731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:31:14.979338195Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:31:16.024151035Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:31:16.024172875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-04T12:31:16.492238273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:48206"
level=debug ts=2026-05-04T12:31:17.037719696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:31:19.441219194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:31:19.441229637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:31:19.44124667Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:31:19.44125523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:31:19.446119475Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-04T12:31:19.447557331Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006150825s
level=warn ts=2026-05-04T12:31:24.441938901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:31:25.145420389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=warn ts=2026-05-04T12:31:29.441518809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:31:34.44177103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:31:39.441501764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:31:41.407943954Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:31:44.442026514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:31:47.665699561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38862"
level=info ts=2026-05-04T12:31:48.153586438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:31:48.162661009Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:31:49.422832973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:31:49.424949342Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:31:49.441305348Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:31:49.441338446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:31:49.441344105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:31:49.441358077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:31:49.441368304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:31:49.441447339Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-04T12:31:49.441454077Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:31:49.441470441Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.318µs success=true
level=debug ts=2026-05-04T12:31:49.441663069Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-04T12:31:50.249026801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:31:50.253792536Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:31:52.826323549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:31:52.833279974Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:31:52.860396618Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T12:31:53.901397126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:31:53.907936891Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:31:54.441583373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:31:55.146935611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:31:56.277516649Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T12:31:58.323373792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:31:58.331828535Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:31:59.441790731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:31:59.695679272Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-04T12:32:03.11277349Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T12:32:04.24791566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:32:04.255505956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:32:04.441988871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:06.531017822Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-04T12:32:09.442127795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:09.948331955Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:32:10.32520805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:32:10.327086488Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:32:13.366691621Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:32:14.441555691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:16.464481985Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:32:16.784256551Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T12:32:17.666594253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44166"
level=info ts=2026-05-04T12:32:19.44151629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:32:19.441543801Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:32:19.441551936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:32:19.441513653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:20.202345202Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-04T12:32:23.619268715Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:32:24.029432213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46378"
level=warn ts=2026-05-04T12:32:24.441707153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:32:25.148736227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:32:27.037359922Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:32:27.037385608Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-05-04T12:32:29.441424974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:30.456054857Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-04T12:32:30.456076909Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-04T12:32:33.872666538Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-04T12:32:34.441546031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:37.290298515Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-04T12:32:39.441719004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:32:39.936917368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:32:39.945865853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:32:40.708386571Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-04T12:32:41.116576613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:32:41.125126422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:32:41.306635525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:32:41.318053746Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:32:44.126291102Z caller=checkpoint.go:372 msg="writing series" size="595 kB"
level=warn ts=2026-05-04T12:32:44.4418348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:32:45.123853963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:32:45.130806874Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:32:47.543947382Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=debug ts=2026-05-04T12:32:47.543977147Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-04T12:32:48.647828595Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:32:48.795098814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:32:48.806029349Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:32:49.423227464Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:32:49.424309272Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:32:49.441802008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:32:49.441826446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:32:49.44183432Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:32:49.441837388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:32:50.961356233Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-04T12:32:53.517112739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:32:53.523947631Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-04T12:32:54.032665466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:39552"
level=debug ts=2026-05-04T12:32:54.379053036Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:32:54.441220197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:32:55.150754493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:32:57.796206951Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-04T12:32:59.441707238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:01.217330651Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-04T12:33:01.217355689Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=warn ts=2026-05-04T12:33:04.441753474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:04.63207482Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T12:33:08.049593383Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T12:33:08.625710989Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:33:09.441782381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:11.477665989Z caller=checkpoint.go:372 msg="writing series" size="9.7 MB"
level=debug ts=2026-05-04T12:33:11.477689613Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.8 MB" series=3
level=warn ts=2026-05-04T12:33:14.442010895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:14.885098154Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-04T12:33:17.670713989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44256"
level=debug ts=2026-05-04T12:33:18.303090965Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-04T12:33:19.441253101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:33:19.441291192Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:33:19.441302752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:33:19.441296562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:21.720815402Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-04T12:33:23.668986435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:33:24.035793622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46984"
level=warn ts=2026-05-04T12:33:24.441761575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:25.13859412Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
ts=2026-05-04T12:33:25.154020546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:33:28.56220709Z caller=checkpoint.go:372 msg="writing series" size="8.4 MB"
level=debug ts=2026-05-04T12:33:28.562229541Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.0 MB" series=5
level=info ts=2026-05-04T12:33:29.343554941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:33:29.349762653Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:33:29.442094268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:33:31.820553235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:33:31.829330197Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:33:31.829355372Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:33:31.829381173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:33:31.837541259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:33:31.973817722Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-04T12:33:32.365113675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:33:32.37216346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:33:34.442125836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:33:35.049600249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:33:35.056438302Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:33:35.391645739Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T12:33:37.30041741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:33:37.30628502Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:33:38.07922815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:33:38.087932106Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:33:38.809218016Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T12:33:39.441833744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:42.226608499Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:33:44.441980863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:45.644738136Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-04T12:33:47.037746633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:33:47.046246028Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:33:49.06234536Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-04T12:33:49.423717682Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:33:49.424861654Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:33:49.441219988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:33:49.441226348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:49.441243969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:33:49.441254732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:33:52.479931721Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T12:33:54.442170372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:33:55.15740467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:33:55.898629549Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=debug ts=2026-05-04T12:33:56.140258016Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:33:59.315251504Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:33:59.315273402Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-04T12:33:59.441809992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:33:59.905662953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:34:02.73303182Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=warn ts=2026-05-04T12:34:04.44211593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:06.150749678Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T12:34:09.273078105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:34:09.282316455Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:34:09.441997218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:09.569364236Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-04T12:34:12.98736076Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=debug ts=2026-05-04T12:34:12.98738452Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-04T12:34:14.441571783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:16.405814628Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-04T12:34:16.405835708Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
level=info ts=2026-05-04T12:34:17.630147811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:34:17.637574023Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:34:19.442027528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:34:19.442060914Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:34:19.442068787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:34:19.442043551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:19.821908539Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T12:34:21.120909072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:34:21.127846716Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:34:23.241139156Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-04T12:34:23.241161444Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=2
level=warn ts=2026-05-04T12:34:24.442245584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:34:24.682247173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:34:24.688121029Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T12:34:25.160289601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:34:26.657889862Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-04T12:34:27.706255419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:34:29.44212742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:30.075596619Z caller=checkpoint.go:372 msg="writing series" size="511 B"
level=info ts=2026-05-04T12:34:32.830619132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:34:32.840047107Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:34:33.493124635Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-04T12:34:34.441695267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:34:34.621649236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:34:34.631342205Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:34:34.631367612Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:34:34.63138977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:34:34.636714472Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:34:36.910308869Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:34:39.441536485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:40.327908745Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:34:43.746286389Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-04T12:34:44.441583094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:34:45.615953992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:34:45.624533775Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T12:34:46.511440127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41482"
level=debug ts=2026-05-04T12:34:47.163635949Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
ts=2026-05-04T12:34:47.674766405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:48328"
level=debug ts=2026-05-04T12:34:49.422999214Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:34:49.425131452Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:34:49.441554673Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:34:49.441573271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:34:49.441582342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:34:49.441640836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:34:50.581011312Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-04T12:34:51.972093059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:34:53.999450708Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T12:34:53.999475278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=9
ts=2026-05-04T12:34:54.044950614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52368"
level=warn ts=2026-05-04T12:34:54.44160386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:34:55.162392209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:34:57.417019712Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=info ts=2026-05-04T12:34:58.482958742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:34:58.49266151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:34:59.134098756Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:34:59.134143823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="681 kB" avg_comp="681 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:34:59.138766712Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:34:59.44140752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:00.835128246Z caller=checkpoint.go:372 msg="writing series" size="979 kB"
level=debug ts=2026-05-04T12:35:00.835150571Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-04T12:35:02.244610352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:35:02.251600744Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:35:04.252100091Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-04T12:35:04.441509766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:35:06.816234626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:35:06.823321454Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:35:06.917900553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:35:06.923429341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:35:07.66951983Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:35:09.44172791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:11.087456092Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=warn ts=2026-05-04T12:35:14.44177039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:14.506290304Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=debug ts=2026-05-04T12:35:14.506310679Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T12:35:14.731737688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:35:14.73693831Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-04T12:35:16.512290841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43724"
level=debug ts=2026-05-04T12:35:17.924428351Z caller=checkpoint.go:372 msg="writing series" size="810 kB"
level=info ts=2026-05-04T12:35:19.441489462Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:35:19.441501318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:19.441532758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:35:19.441542272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:35:20.901354922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:35:20.906375626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:35:21.34064218Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-04T12:35:23.112409647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:35:23.120279557Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:35:24.44178364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:24.758589478Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=debug ts=2026-05-04T12:35:24.758611609Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-04T12:35:25.16506807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:35:28.176218186Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-05-04T12:35:29.441790295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:31.59459921Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-04T12:35:34.138761934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:35:34.441472046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:35.0117649Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=debug ts=2026-05-04T12:35:38.43017843Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:35:39.441809534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:41.88687792Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-04T12:35:41.88690316Z caller=checkpoint.go:384 msg="flushing series" totalSize="49 MB" series=5
level=warn ts=2026-05-04T12:35:44.441323725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:35:45.265130161Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
ts=2026-05-04T12:35:46.513302304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44844"
ts=2026-05-04T12:35:47.679679274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42844"
level=debug ts=2026-05-04T12:35:48.68325516Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=info ts=2026-05-04T12:35:49.42435908Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:35:49.424956129Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:35:49.441276885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:35:49.44129774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:35:49.44130569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:35:49.441455301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:35:51.071079511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:35:51.07720764Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T12:35:51.944823976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:35:51.952668312Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:35:52.100262183Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-04T12:35:52.898769089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:35:52.903914365Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:35:53.47406751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:35:54.44183139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:35:55.167902014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:35:55.179577217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:35:55.186836676Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:35:55.518226975Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T12:35:55.518249219Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T12:35:57.728223813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:35:57.73493143Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:35:57.734954014Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:35:57.734976311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:35:57.740854432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:35:58.936445971Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:35:59.441787835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:02.353469811Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=debug ts=2026-05-04T12:36:02.957325339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:36:04.441296252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:05.770829438Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-04T12:36:09.189051821Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-04T12:36:09.442117334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:36:10.256416617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:36:10.266093624Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:36:10.292272741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:36:10.296826092Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:36:12.607054559Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-04T12:36:13.198628095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:36:13.207312869Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:36:14.441361273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:16.024837778Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:36:16.024860082Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-04T12:36:16.51480351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47996"
level=info ts=2026-05-04T12:36:19.441374169Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:36:19.441382412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:19.441401441Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:36:19.441409669Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:36:19.448364142Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-04T12:36:19.44983172Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008250383s
ts=2026-05-04T12:36:24.054639785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:48212"
level=warn ts=2026-05-04T12:36:24.442143373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:36:25.169310246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=warn ts=2026-05-04T12:36:29.441502897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:36:33.229370053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:36:33.237312078Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:36:34.442124425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:36.226985423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:36:38.883578521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:36:38.889892492Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:36:39.442057473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:36:39.525465541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:36:39.534402378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:36:44.441723222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:45.142776611Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:36:45.144736048Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T12:36:47.380050593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:36:47.386652193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:36:49.423560039Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:36:49.424629474Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:36:49.441050441Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:36:49.441151959Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-04T12:36:49.441160136Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:36:49.441176526Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.705µs success=true
level=info ts=2026-05-04T12:36:49.441239099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:36:49.441242163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:49.441260368Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:36:49.441270792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:36:49.441372885Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-04T12:36:49.519106968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:36:49.526070888Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:36:49.526096369Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:36:49.526129096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:36:49.531060505Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:36:52.860013255Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T12:36:54.441470724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:36:55.171278979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:36:56.277221297Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T12:36:59.442158411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:36:59.69517358Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-04T12:37:00.498495091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:37:00.505788836Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:37:01.495680518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:37:01.504046019Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:37:01.504068835Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:37:01.504096667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:37:01.512305326Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:37:01.990620564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:37:01.995569195Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:37:01.995592201Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:37:01.995615187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:37:02.000114454Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:37:03.11227824Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:37:04.441716177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:37:06.530907078Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-05-04T12:37:09.441398694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:37:09.948046341Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:37:10.433166006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:37:13.365813757Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:37:14.441386793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:37:16.515676491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:37:16.527588121Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:37:16.783182113Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:37:19.441634651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:37:19.441633995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:37:19.441666697Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:37:19.441677513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:37:20.2019072Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=info ts=2026-05-04T12:37:21.784742505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:37:21.790199554Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:37:21.828574627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:37:21.835217823Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:37:23.61889627Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:37:24.061716128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:54234"
level=warn ts=2026-05-04T12:37:24.442098629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:37:25.172992838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:37:27.037017533Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:37:27.037042145Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=warn ts=2026-05-04T12:37:29.44191743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:37:30.174751309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:37:30.182019768Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:37:30.456926073Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-04T12:37:30.456948379Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-04T12:37:31.000513754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:37:31.00849408Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:37:32.983749849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:37:32.988773988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:37:33.872127201Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=debug ts=2026-05-04T12:37:34.361679105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:37:34.441689469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:37:37.289965416Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-04T12:37:39.441494153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:37:39.5528289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:37:39.559600299Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:37:40.130861454Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:37:40.707621545Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=info ts=2026-05-04T12:37:43.339521941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:37:43.344526964Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:37:43.845790217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:37:43.854438024Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:37:44.126529608Z caller=checkpoint.go:372 msg="writing series" size="727 kB"
level=debug ts=2026-05-04T12:37:44.126552566Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-04T12:37:44.44145875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:37:47.542970293Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
ts=2026-05-04T12:37:47.686792346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44484"
level=debug ts=2026-05-04T12:37:49.423179189Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:37:49.424353198Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:37:49.441867417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:37:49.44188867Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:37:49.441896478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:37:49.441965543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:37:50.9614371Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
ts=2026-05-04T12:37:54.0645496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:60076"
level=debug ts=2026-05-04T12:37:54.378821576Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:37:54.441255577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:37:55.174061188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:37:57.796333529Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-04T12:37:59.442124446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:01.217722585Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-04T12:38:01.217749497Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=5
level=info ts=2026-05-04T12:38:02.200070534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:38:02.206592031Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:38:04.441275354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:04.632264659Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T12:38:08.049425655Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:38:09.44122361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:10.816714558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:38:11.476093224Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-04T12:38:11.476112593Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=3
level=warn ts=2026-05-04T12:38:14.442194197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:14.885029075Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-04T12:38:17.688258252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42192"
level=debug ts=2026-05-04T12:38:18.302159421Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=warn ts=2026-05-04T12:38:19.441574437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:38:19.441572831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:38:19.44162161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:38:19.441630764Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:38:20.880532475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:38:20.887843808Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:38:21.720467229Z caller=checkpoint.go:372 msg="writing series" size="435 kB"
level=info ts=2026-05-04T12:38:23.493847398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:38:23.500825323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:38:24.441248364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:25.137828389Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
ts=2026-05-04T12:38:25.174989649Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:38:25.736115655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:38:27.399237111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:38:27.405862551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:38:28.565120201Z caller=checkpoint.go:372 msg="writing series" size="9.0 MB"
level=debug ts=2026-05-04T12:38:28.565143799Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=5
level=info ts=2026-05-04T12:38:28.678721078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:38:28.682894285Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:38:29.441927228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:31.973741989Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-04T12:38:34.029749558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:38:34.034028193Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:38:34.441575899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:35.391264728Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T12:38:38.809042655Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T12:38:39.441890272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:38:39.810872507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:38:39.819945987Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:38:42.226746705Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:38:44.441610431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:45.644224015Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-04T12:38:46.032729742Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T12:38:46.526860541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:44738"
level=debug ts=2026-05-04T12:38:49.061762644Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T12:38:49.423805037Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:38:49.424866517Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:38:49.441209334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:38:49.441228748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:38:49.441236483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:38:49.441323294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:38:52.479564785Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-04T12:38:54.441335049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:38:55.177674274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:38:55.897658433Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-04T12:38:58.881845366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:38:58.887502659Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:38:59.315148751Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:38:59.315173763Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-04T12:38:59.441552503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:02.733476874Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=warn ts=2026-05-04T12:39:04.441430944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:39:06.07837842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:39:06.084898383Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:39:06.151120537Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T12:39:08.930846354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:39:08.935519426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:39:09.441196292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:09.568907395Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-04T12:39:12.986467632Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=debug ts=2026-05-04T12:39:12.986493104Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-04T12:39:14.441317738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:15.024251299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:39:15.079290231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:39:15.085701019Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:39:16.405007021Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-04T12:39:16.405026919Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
level=info ts=2026-05-04T12:39:18.152230865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:39:18.156790927Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:39:19.441771154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:39:19.441799165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:39:19.441807429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:39:19.441838352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:19.822080055Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T12:39:20.229682159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:39:20.23724773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:39:20.709739341Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:39:23.241001427Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-04T12:39:23.241023314Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=2
level=warn ts=2026-05-04T12:39:24.441683479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:39:25.180990872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:39:26.656775097Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-04T12:39:29.442204034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:30.075098987Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=debug ts=2026-05-04T12:39:33.492682457Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
level=warn ts=2026-05-04T12:39:34.44203142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:39:35.039648642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:39:35.052156518Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:39:36.910626453Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-04T12:39:39.441979865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:40.328150482Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:39:43.746092385Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-04T12:39:44.442099216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:47.163640314Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-04T12:39:47.19472953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:39:47.202724008Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:39:47.23610184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:39:49.423569515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:39:49.424613611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:39:49.442079276Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:39:49.442088498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:39:49.442103289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:39:49.442112606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:39:50.581605394Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=debug ts=2026-05-04T12:39:54.000027263Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T12:39:54.000075138Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=9
level=info ts=2026-05-04T12:39:54.169455136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:39:54.175160907Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:39:54.441964952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:39:55.182690722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:39:57.416867068Z caller=checkpoint.go:372 msg="writing series" size="391 kB"
level=warn ts=2026-05-04T12:39:59.441260038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:39:59.846832992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:39:59.851095956Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:40:00.492091242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:40:00.834847168Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-04T12:40:00.834868652Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-04T12:40:01.009440734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:40:01.013402189Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:40:04.25246922Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-04T12:40:04.441363496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:40:06.035296639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:40:06.04412556Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:40:06.04414607Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:40:06.044171112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:40:06.050246801Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:40:07.66991666Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:40:09.441994213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:11.087520365Z caller=checkpoint.go:372 msg="writing series" size="353 kB"
level=warn ts=2026-05-04T12:40:14.441271752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:14.505906228Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-04T12:40:14.505929182Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T12:40:15.015235246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:40:15.021518307Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:40:17.129764557Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:40:17.923407539Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=info ts=2026-05-04T12:40:18.2508162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:40:18.258730921Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:40:19.441952987Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:40:19.441980831Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:40:19.441989404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:40:19.442037107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:40:20.594236835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:40:20.602395014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:40:21.340699004Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-04T12:40:24.441317173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:24.758500235Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-04T12:40:24.758519395Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-05-04T12:40:25.184438131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:40:28.176306153Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-04T12:40:29.442066551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:31.59395089Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T12:40:34.441511093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:35.011927657Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-04T12:40:38.429158171Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:40:39.441581305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:41.889289762Z caller=checkpoint.go:372 msg="writing series" size="50 MB"
level=debug ts=2026-05-04T12:40:41.88933527Z caller=checkpoint.go:384 msg="flushing series" totalSize="50 MB" series=5
level=warn ts=2026-05-04T12:40:44.441978021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:45.264425243Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-05-04T12:40:45.602147239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:40:45.610401346Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:40:45.610424084Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:40:45.610445552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:40:45.616252213Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:40:48.682550982Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-04T12:40:49.423081953Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:40:49.425207567Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:40:49.441518343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:40:49.44152657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:40:49.44154204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:40:49.441549711Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:40:50.338327111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:40:51.39911777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:40:51.405711028Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:40:52.100675688Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-04T12:40:53.155824005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:40:53.161167201Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:40:54.441389307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:40:55.185635122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:40:55.517791291Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T12:40:55.517810928Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T12:40:56.524169176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:40:56.534403489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:40:58.936308084Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=info ts=2026-05-04T12:40:58.938205395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:40:58.944389076Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:40:59.442200596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:41:00.698296663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:41:00.702941422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:41:02.037596478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:41:02.039151841Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:41:02.353037258Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:41:04.441602142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:41:05.770849184Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T12:41:07.194905075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:41:07.20349004Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:41:08.26462436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:41:08.272178767Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:41:09.188487821Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:41:09.441299843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:41:12.606329884Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-04T12:41:14.441546763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:41:16.024607033Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:41:16.024630811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T12:41:19.441249077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:41:19.441245244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:41:19.441278065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:41:19.441288333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:41:19.447747562Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-04T12:41:19.449284992Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007977064s
ts=2026-05-04T12:41:24.086601713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56478"
level=warn ts=2026-05-04T12:41:24.442232905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:41:25.188336634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:41:26.155585847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:41:29.441747449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:41:32.269822526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:41:32.281958543Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:41:33.378140447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:41:33.383008019Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-04T12:41:33.383026481Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:41:33.383048187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:41:33.38699282Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:41:34.441634228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:41:39.441705041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:41:42.780785208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:41:43.838680426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:41:43.844516684Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:41:44.441711451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:41:48.383315683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:41:48.388374848Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:41:49.423165532Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:41:49.425271991Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:41:49.441555116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-04T12:41:49.44156408Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-04T12:41:49.441574604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:41:49.441582803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:41:49.441604964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:41:49.441684792Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-04T12:41:49.441692902Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:41:49.441709223Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.511µs success=true
level=debug ts=2026-05-04T12:41:49.441909889Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-04T12:41:51.880975221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.3 MB" avg_comp="1.3 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:41:51.881028123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-04T12:41:51.887026318Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:41:52.860258275Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T12:41:53.099670283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:41:53.107989022Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:41:54.442216567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:41:55.191124247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:41:55.964934502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:41:55.972324339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:41:56.278240446Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T12:41:59.441883729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:41:59.695675579Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-04T12:42:00.301376895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:42:00.305809231Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:42:02.917439386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:42:02.922918542Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:42:03.113648303Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:42:04.441902477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:42:04.620654328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:42:04.626642368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:42:06.530948811Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-04T12:42:09.441211806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:42:09.948969163Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:42:13.366612286Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-04T12:42:14.29203018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:42:14.299545122Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:42:14.442145992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:42:16.545161512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:56124"
level=debug ts=2026-05-04T12:42:16.784198862Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:42:19.441584638Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:42:19.441581661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:42:19.44161017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:42:19.441618713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:42:20.201570271Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-04T12:42:23.620041703Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:42:24.092891662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:58610"
level=warn ts=2026-05-04T12:42:24.44216464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:42:25.118730958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:42:25.126963463Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-04T12:42:25.193534188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:42:26.198498805Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:42:27.037405689Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:42:27.037426784Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=12
level=info ts=2026-05-04T12:42:28.688226082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:42:28.694880212Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:42:29.442143664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:42:30.457087053Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T12:42:30.457109099Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-04T12:42:31.835375092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:42:31.840840691Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:42:33.872755357Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-04T12:42:34.441685581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:42:36.300898526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:42:36.310136556Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:42:37.29002662Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-04T12:42:39.442128826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:42:40.708663988Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-04T12:42:44.126712749Z caller=checkpoint.go:372 msg="writing series" size="865 kB"
level=debug ts=2026-05-04T12:42:44.126734641Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-04T12:42:44.441741101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:42:44.907439772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T12:42:46.545979276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53910"
level=info ts=2026-05-04T12:42:46.664097966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:42:46.669077204Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:42:47.544072233Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
ts=2026-05-04T12:42:47.70417594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:53750"
level=debug ts=2026-05-04T12:42:49.423020435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:42:49.425140678Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:42:49.441353077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:42:49.441365898Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:42:49.441363778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:42:49.441375021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:42:50.961798131Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-04T12:42:51.775004918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:42:51.782504297Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T12:42:54.096383267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:40798"
level=debug ts=2026-05-04T12:42:54.378623498Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:42:54.441820302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:42:55.194835924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T12:42:55.875544163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:42:55.882637624Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:42:55.882658764Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:42:55.882681248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:42:55.888796363Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:42:57.797118748Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-04T12:42:58.196389045Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:42:59.441280861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:01.216378795Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-04T12:43:01.216460699Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=5
level=warn ts=2026-05-04T12:43:04.441850121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:04.632667778Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T12:43:08.050096261Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T12:43:08.382958932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:43:08.392921899Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T12:43:09.362929156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:43:09.368588521Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T12:43:09.441765253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:11.477669609Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T12:43:11.477690475Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T12:43:13.21979369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:43:13.226165249Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:43:14.441363358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:14.885678957Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-04T12:43:16.663785426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:43:16.672099684Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T12:43:17.705925983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42874"
level=debug ts=2026-05-04T12:43:18.302774057Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-04T12:43:19.442124695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:43:19.442136408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:19.442151537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:43:19.442162282Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:43:21.041816571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:43:21.720905248Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=warn ts=2026-05-04T12:43:24.441302144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:25.139141056Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
ts=2026-05-04T12:43:25.196523791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T12:43:26.393901256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:43:26.399626424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:43:26.39965713Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:43:26.399681489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:43:26.404697443Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:43:28.559232788Z caller=checkpoint.go:372 msg="writing series" size="9.7 MB"
level=debug ts=2026-05-04T12:43:28.559256514Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=5
level=warn ts=2026-05-04T12:43:29.441879027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:29.72687415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:43:30.601048624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:43:30.608417571Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:43:30.60843664Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:43:30.608460402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:43:30.615177566Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:43:31.973986604Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-04T12:43:34.441575502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:35.392018588Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T12:43:37.62295554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:43:37.627792595Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:43:37.62781345Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:43:37.627836664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:43:37.631718088Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:43:38.809355681Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T12:43:39.441235484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:43:41.036756883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:43:41.043559021Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:43:42.227067601Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-04T12:43:42.256174553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:43:42.2615263Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:43:43.064790527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:43:43.07207656Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:43:44.441446824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:43:45.644611441Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-04T12:43:47.707309002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:54650"
level=debug ts=2026-05-04T12:43:49.063107328Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-04T12:43:49.195402034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:43:49.200702891Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:43:49.423352708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:43:49.424402475Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:43:49.44176678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:43:49.441778605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:43:49.44178524Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:43:49.441837124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:43:50.357546864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:43:50.366282234Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:43:52.48062149Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T12:43:53.646538291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:43:53.65193839Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T12:43:54.441294468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:43:55.198888775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:43:55.898617611Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-04T12:43:56.511114261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:43:56.51601289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:43:59.316178034Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:43:59.316213278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-04T12:43:59.441806974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:02.733914432Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-05-04T12:44:02.803956754Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:44:03.92611646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:44:03.932938971Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:44:04.441732561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:06.151430254Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-04T12:44:09.442032998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:09.569537557Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-04T12:44:12.830921136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:44:12.838210401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:44:12.986718635Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=debug ts=2026-05-04T12:44:12.986738469Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-04T12:44:14.30221986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:44:14.311125009Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:44:14.441429093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:16.405529328Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-04T12:44:16.405553227Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
ts=2026-05-04T12:44:17.70882781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:51336"
level=info ts=2026-05-04T12:44:18.361942213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:44:18.368549987Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:44:19.441502684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:44:19.441505374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:44:19.441538351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:44:19.441547807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:44:19.822605108Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-04T12:44:20.856074707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:44:23.242007245Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-04T12:44:23.242029042Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-04T12:44:23.663414373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:44:23.672229096Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T12:44:24.106351089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:57128"
level=warn ts=2026-05-04T12:44:24.441315993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:44:25.20231878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:44:26.657599417Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-04T12:44:28.033068732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:44:28.039037406Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:44:28.407941463Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:44:29.442059089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:30.075247462Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=debug ts=2026-05-04T12:44:33.493586423Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-05-04T12:44:34.441233052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:36.910293295Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-04T12:44:39.441222959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:40.328902434Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:44:43.745800048Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-04T12:44:44.442169679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:44:45.501460616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:44:45.507803001Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:44:45.615039352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:44:45.61976533Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:44:47.163855281Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-04T12:44:49.423168983Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:44:49.425270061Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:44:49.441566353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:44:49.441601418Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:44:49.441610191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:44:49.441630147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:44:50.581775123Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:44:54.000363942Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T12:44:54.00038957Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=warn ts=2026-05-04T12:44:54.44154793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:44:54.816169143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:44:54.824919042Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T12:44:55.204499317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T12:44:55.506169839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:44:55.514230233Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:44:57.417419079Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:44:59.441913525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:00.834705775Z caller=checkpoint.go:372 msg="writing series" size="954 kB"
level=debug ts=2026-05-04T12:45:00.8347334Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-05-04T12:45:02.788232073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:45:02.789677763Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:45:04.140178956Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:45:04.252628493Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-04T12:45:04.442066475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:45:07.380949961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:45:07.387275396Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:45:07.670272561Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-04T12:45:08.51562842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:45:08.519488572Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:45:09.441901448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:11.087875141Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=info ts=2026-05-04T12:45:12.349973191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:45:12.356234477Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:45:14.441752818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:14.505610163Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-04T12:45:14.50563508Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
ts=2026-05-04T12:45:16.55861688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60644"
ts=2026-05-04T12:45:17.710833354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:45902"
level=info ts=2026-05-04T12:45:17.72158604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:45:17.728770895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:45:17.923596506Z caller=checkpoint.go:372 msg="writing series" size="831 kB"
level=info ts=2026-05-04T12:45:19.441932229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:45:19.441964314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:45:19.441974612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:45:19.441997589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:21.34107103Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
ts=2026-05-04T12:45:24.112040526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59806"
level=warn ts=2026-05-04T12:45:24.441678426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:24.75886262Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-04T12:45:24.758883523Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-04T12:45:25.20691108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:45:26.862238992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:45:26.871151705Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:45:28.176662346Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-04T12:45:29.442170237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:30.988850801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:45:31.594536136Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T12:45:34.441361441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:35.012712842Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=info ts=2026-05-04T12:45:37.187666714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:45:37.193731487Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:45:38.429986336Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:45:39.441322334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:41.890973141Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-04T12:45:41.891000178Z caller=checkpoint.go:384 msg="flushing series" totalSize="52 MB" series=5
level=warn ts=2026-05-04T12:45:44.441719934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:45:45.265065835Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
ts=2026-05-04T12:45:46.560353726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46748"
level=info ts=2026-05-04T12:45:48.474133947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:45:48.484331892Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:45:48.683874485Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-04T12:45:49.422989377Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:45:49.425206157Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T12:45:49.42540597Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T12:45:49.425443959Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975442 n=1
level=debug ts=2026-05-04T12:45:49.425490383Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975442 n=1
level=info ts=2026-05-04T12:45:49.441496355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:45:49.441515517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:45:49.44150669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:45:49.441523149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:45:49.46683042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:45:49.473412607Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:45:50.205769335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:45:50.212724227Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:45:52.100177043Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-04T12:45:54.441780702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:45:55.208907692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:45:55.518175606Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T12:45:55.518220054Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-04T12:45:55.804707186Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:45:55.832375543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:45:56.452077314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:45:56.458943083Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:45:58.936107169Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=info ts=2026-05-04T12:45:59.418835977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:45:59.423451309Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:45:59.441709444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:02.354271173Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=warn ts=2026-05-04T12:46:04.441320262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:05.771629651Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T12:46:09.189072684Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-04T12:46:09.44217515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:12.606801198Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-05-04T12:46:14.35239635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:46:14.362770226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:46:14.442085258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:16.025598996Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:46:16.025620665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-04T12:46:16.561564786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:54058"
level=info ts=2026-05-04T12:46:19.441910774Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:46:19.441921385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:19.441937498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:46:19.441946047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:46:19.448605125Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-04T12:46:19.450178626Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008344277s
level=info ts=2026-05-04T12:46:19.510212782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:46:19.516557686Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:46:21.941986655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:46:21.94751143Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:46:24.441889374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:46:25.210901327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=warn ts=2026-05-04T12:46:29.44145167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:30.844204473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:46:34.441351148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:46:35.940976443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:46:35.950012134Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:46:36.429004636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:46:36.435586308Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:46:39.069923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:46:39.075934314Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:46:39.410462271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:46:39.44119741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:46:44.441217724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:46:45.189811999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:46:45.197277831Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T12:46:47.716064552Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:34628"
level=debug ts=2026-05-04T12:46:49.423387498Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:46:49.424426421Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:46:49.441793075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:46:49.441808007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:46:49.441815625Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:46:49.441805716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:46:49.441823135Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:46:49.441940157Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-04T12:46:49.441946547Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:46:49.441963345Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.099µs success=true
level=debug ts=2026-05-04T12:46:49.442169744Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-04T12:46:50.524225723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:46:50.528633876Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:46:52.860314931Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T12:46:54.441311371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:46:55.21216037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:46:56.278208769Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T12:46:56.728585659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:46:56.736822652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:46:56.7368481Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:46:56.736875086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:46:56.744364997Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:46:59.441592952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:46:59.69570257Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-04T12:47:03.11354203Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-04T12:47:04.4415747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:05.23353122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:47:06.531780453Z caller=checkpoint.go:372 msg="writing series" size="303 kB"
level=info ts=2026-05-04T12:47:06.887542817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:47:06.893951592Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:47:09.441600453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:09.948804315Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-04T12:47:13.366875119Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:47:14.442133142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:47:15.377914751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:47:15.383471112Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:47:16.509128693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:47:16.515806749Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T12:47:16.566326227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:43550"
level=debug ts=2026-05-04T12:47:16.784046776Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-04T12:47:17.717335869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56276"
level=info ts=2026-05-04T12:47:19.441388975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:47:19.441416396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:47:19.441424252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:47:19.441425311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:20.202988829Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=debug ts=2026-05-04T12:47:20.203013306Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-04T12:47:21.913888032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:47:21.919845438Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:47:23.619642067Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:47:24.125256717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:51548"
level=warn ts=2026-05-04T12:47:24.441589445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:47:25.21401708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:47:27.037538899Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=info ts=2026-05-04T12:47:28.157489663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:47:28.166996275Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:47:29.441701784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:30.457118039Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-04T12:47:30.457145824Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=debug ts=2026-05-04T12:47:31.033001794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:47:31.137887933Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:47:33.87315154Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=warn ts=2026-05-04T12:47:34.441615673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:47:34.640836526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:47:34.647357971Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:47:34.647382207Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:47:34.647405357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:47:34.653366702Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:47:36.850833476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:47:36.859562263Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:47:37.290449393Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-04T12:47:38.033779351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:47:38.038360718Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:47:39.442095641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:40.708490347Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-04T12:47:44.126997955Z caller=checkpoint.go:372 msg="writing series" size="961 kB"
level=debug ts=2026-05-04T12:47:44.127021631Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-04T12:47:44.441976158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:47.54380275Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-04T12:47:47.619031757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:47:47.626139629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T12:47:47.718500047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:37210"
level=debug ts=2026-05-04T12:47:49.423695134Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:47:49.424752533Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:47:49.442121851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:47:49.442133199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:49.442148318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:47:49.442159881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:47:50.961506166Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T12:47:50.96237794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:47:50.969488221Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T12:47:54.128125155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:51544"
level=debug ts=2026-05-04T12:47:54.379087246Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-04T12:47:54.44166486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:47:55.215405364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:47:57.797435667Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-04T12:47:59.442047254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:47:59.681775111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:48:00.100418295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:48:00.106557368Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:48:00.106581651Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:48:00.106610871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:48:00.112360102Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:48:01.150023445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:48:01.155358399Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-04T12:48:01.155383969Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:48:01.155409425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:48:01.15975024Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:48:01.217441475Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-04T12:48:01.217463664Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=5
level=info ts=2026-05-04T12:48:02.807816047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:48:02.814014756Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:48:04.44162338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:04.633026335Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-04T12:48:08.049981291Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:48:09.442071662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:11.480821653Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T12:48:11.480853295Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T12:48:11.976308606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:48:11.982727885Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T12:48:14.441513266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:14.885650367Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-04T12:48:16.257177172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:48:16.261581764Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T12:48:17.435615924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:48:17.443269665Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T12:48:17.719683093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:34054"
level=debug ts=2026-05-04T12:48:17.799912647Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:48:18.303331537Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-04T12:48:18.441065267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:48:18.446933206Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:48:19.441996716Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:48:19.44202463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:48:19.442041513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:48:19.442050988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:21.721625517Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=warn ts=2026-05-04T12:48:24.441426436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:25.139355647Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
ts=2026-05-04T12:48:25.217925143Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:48:27.298406826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:48:27.304731221Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:48:28.567291504Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-04T12:48:28.567316836Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=5
level=info ts=2026-05-04T12:48:28.900166345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:48:28.904798455Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:48:29.44212563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:31.97403031Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-04T12:48:32.71122472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:48:32.718400498Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:48:34.441619448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:35.389547313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:48:35.391588811Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T12:48:38.81054103Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T12:48:39.441978749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:48:41.989000852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:48:41.997300665Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:48:42.227360952Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:48:44.441625266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:45.645599179Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-04T12:48:47.721072638Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:33136"
level=info ts=2026-05-04T12:48:48.748008048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:48:48.753794728Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:48:49.063406238Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T12:48:49.423240643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:48:49.424273039Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:48:49.44166538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:48:49.44167902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:48:49.441689884Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:48:49.441697821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:48:52.48091298Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-04T12:48:54.442242353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:48:55.219820094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:48:55.850310215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:48:55.8553902Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:48:55.89884728Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=debug ts=2026-05-04T12:48:59.316078269Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:48:59.316098843Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-04T12:48:59.44180967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:48:59.757453028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:48:59.76546429Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:49:02.734317575Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=warn ts=2026-05-04T12:49:04.442102751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:05.666322557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:49:05.66823141Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:49:06.151637687Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-04T12:49:09.061019241Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:49:09.441393037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:09.569550557Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-04T12:49:10.443906797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:49:10.451868956Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:49:10.45189529Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:49:10.451922176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:49:10.457910531Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:49:12.98747436Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=debug ts=2026-05-04T12:49:12.987503055Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-04T12:49:14.441692007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:49:16.36598395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:49:16.373031496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:49:16.405879996Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-04T12:49:16.405902093Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=info ts=2026-05-04T12:49:17.420343069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:49:17.426344003Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:49:19.442144278Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:49:19.442156899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:19.442171077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:49:19.442178816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:49:19.822647348Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-04T12:49:23.242089949Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-04T12:49:23.242112956Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=info ts=2026-05-04T12:49:23.345997153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:49:23.352482143Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:49:23.526167586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:49:23.530163455Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:49:24.441784785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:49:25.221608006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:49:26.657639473Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-05-04T12:49:29.44163484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:30.075852127Z caller=checkpoint.go:372 msg="writing series" size="627 B"
level=debug ts=2026-05-04T12:49:33.452182147Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:49:33.4936539Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=warn ts=2026-05-04T12:49:34.441378673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:36.911085469Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-04T12:49:37.878941717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:49:37.887461248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:49:37.887489394Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:49:37.887517422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:49:37.892602222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:49:39.441353874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:40.328535468Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-04T12:49:41.447052826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:49:41.452780826Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:49:43.747070838Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-04T12:49:44.441212203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:47.164293349Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
ts=2026-05-04T12:49:47.723546014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:56954"
level=debug ts=2026-05-04T12:49:49.422930254Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:49:49.425210236Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:49:49.441507379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:49:49.441508219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:49:49.441529857Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:49:49.441538342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:49:50.58151652Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-04T12:49:52.731704654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:49:52.738332581Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:49:53.282894612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:49:53.288431455Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:49:54.000313708Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T12:49:54.000336141Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=9
level=warn ts=2026-05-04T12:49:54.441542852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:49:54.842373559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:49:54.847254413Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-04T12:49:55.224514187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:49:57.417869344Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=info ts=2026-05-04T12:49:58.678137721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:49:58.686531933Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:49:58.686557843Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:49:58.686587379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:49:58.694711863Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:49:59.441715462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:00.835308483Z caller=checkpoint.go:372 msg="writing series" size="913 kB"
level=debug ts=2026-05-04T12:50:00.835332169Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-05-04T12:50:02.579095998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:50:03.188674888Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:50:04.25311225Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-04T12:50:04.441264726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:50:05.024748237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:50:05.030630693Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:50:07.671036192Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-04T12:50:09.441909685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:50:10.829469707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:50:10.83665109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:50:11.08842621Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=info ts=2026-05-04T12:50:11.476531413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:50:11.480658064Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:50:11.4806827Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:50:11.480704747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:50:11.484291624Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:50:14.12378018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:50:14.129303562Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:50:14.441213996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:14.506756836Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-04T12:50:14.50677573Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-04T12:50:17.725376623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:38502"
level=debug ts=2026-05-04T12:50:17.924014306Z caller=checkpoint.go:372 msg="writing series" size="859 kB"
level=info ts=2026-05-04T12:50:18.210136703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:50:18.221345201Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:50:19.442122283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:50:19.44213609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:19.442155463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:50:19.442165837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:50:21.342005497Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-04T12:50:24.144367159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:53056"
level=warn ts=2026-05-04T12:50:24.441645423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:24.759180122Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=debug ts=2026-05-04T12:50:24.759221938Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-04T12:50:25.227079741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:50:26.550730276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:50:26.556913695Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:50:28.176640079Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-04T12:50:29.441219433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:30.544979287Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:50:31.435639525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:50:31.440104967Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:50:31.59446622Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T12:50:34.133502742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:50:34.139665113Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:50:34.441456197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:35.012065575Z caller=checkpoint.go:372 msg="writing series" size="505 kB"
level=debug ts=2026-05-04T12:50:38.430615202Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:50:39.441410305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:41.891983882Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-04T12:50:41.892019288Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=5
level=warn ts=2026-05-04T12:50:44.442260546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:50:44.557164211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:50:44.564593615Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-04T12:50:44.609711422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:50:44.619500921Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:50:45.265120906Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
ts=2026-05-04T12:50:46.582601411Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:41518"
level=debug ts=2026-05-04T12:50:48.683460239Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-04T12:50:49.42328587Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:50:49.424341352Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:50:49.441718077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:50:49.441739565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:50:49.44174703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:50:49.441761139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:50:52.100683584Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-05-04T12:50:54.148083588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:41200"
level=warn ts=2026-05-04T12:50:54.441235637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:50:55.229140094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:50:55.519240713Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T12:50:55.519261058Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-04T12:50:57.543361109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:50:58.849924952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:50:58.855358396Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:50:58.936347399Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:50:59.441225003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:00.906355613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:51:00.91252042Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:51:02.35417781Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-04T12:51:02.436735971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:51:02.442501324Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:51:03.133514192Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:51:04.441628353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:05.546471818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:51:05.554709893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:51:05.772299278Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-04T12:51:09.189319445Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-04T12:51:09.441218153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:51:12.607416082Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-04T12:51:14.441388446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:15.813239837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:51:15.820783048Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:51:16.025844805Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:51:16.025873063Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T12:51:16.031503451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:51:16.037708395Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:51:19.441661901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:51:19.44171264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:51:19.441706763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:19.441724993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:51:19.449678844Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-04T12:51:19.451472823Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00936169s
level=warn ts=2026-05-04T12:51:24.442116682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:51:25.232467429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T12:51:29.369811251Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:51:29.442229551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:31.072950082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:51:31.082339684Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:51:34.441317704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:37.044102618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:51:37.049282111Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:51:39.441967227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:51:39.865521353Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:51:44.44159293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:45.406115393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:51:45.411697624Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-04T12:51:47.72937684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:34914"
level=info ts=2026-05-04T12:51:48.284204959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:51:48.29032046Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:51:49.423396571Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:51:49.424432899Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:51:49.440820658Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:51:49.441011415Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-04T12:51:49.441020553Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:51:49.441038295Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.022µs success=true
level=info ts=2026-05-04T12:51:49.441216495Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:51:49.441248617Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:51:49.441257533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:51:49.441280417Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-04T12:51:49.441307328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:51:50.037855505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:51:50.042120662Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:51:52.86004742Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T12:51:54.441446591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:51:55.234941462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:51:55.458325701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.9 MB" avg_comp="2.6 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:51:55.469669771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:51:56.276883829Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T12:51:59.441889854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:51:59.695074661Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-04T12:52:03.112684459Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-04T12:52:03.698058666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:52:03.698119932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:52:03.704653825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T12:52:04.441224388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:06.530696274Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-04T12:52:06.863448545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:52:06.870892983Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:52:09.442135515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:09.948559988Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T12:52:12.50555455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:52:12.51186481Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:52:13.366241751Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-04T12:52:14.442131824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:52:14.707448255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:52:14.71537467Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:52:16.783698166Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-04T12:52:16.83010106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:52:16.836151743Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T12:52:19.441235496Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:52:19.441265485Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:52:19.441273785Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:52:19.441352787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:19.851920764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:52:20.201567184Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-04T12:52:23.619481718Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
ts=2026-05-04T12:52:24.157047159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:49342"
level=warn ts=2026-05-04T12:52:24.441801449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:52:25.236470219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:52:26.162810656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:52:26.17013075Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:52:27.037257885Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:52:27.03728143Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=info ts=2026-05-04T12:52:28.435877538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:52:28.442772202Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:52:29.441653234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:30.457431524Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-04T12:52:30.457466733Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-04T12:52:31.108313302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:52:31.113453806Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-04T12:52:33.872662989Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-04T12:52:34.442247045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:37.056353213Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:52:37.056365872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="29 kB" avg_uncomp="29 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:52:37.060542373Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T12:52:37.290211722Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T12:52:37.356417215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:52:37.360025078Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:52:37.657127283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="48 kB" avg_uncomp="48 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:52:37.661158324Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T12:52:37.956797968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:52:37.960289688Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T12:52:38.353253431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:52:38.357813931Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T12:52:39.441255665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:40.707805063Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-04T12:52:42.861115518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:52:42.867160979Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T12:52:43.727720442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:52:43.734942646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:52:44.12563201Z caller=checkpoint.go:372 msg="writing series" size="829 kB"
level=debug ts=2026-05-04T12:52:44.125653636Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-04T12:52:44.441245703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:47.54354797Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-04T12:52:49.230537305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:52:49.23452093Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T12:52:49.423748603Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:52:49.424803115Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:52:49.441138784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:52:49.441180781Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:52:49.441205249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:52:49.441200342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:52:50.961438698Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-05-04T12:52:51.272312855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:52:51.276434631Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T12:52:51.315549025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:52:51.319036656Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T12:52:52.015009853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:52:52.01877528Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T12:52:54.378276421Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T12:52:54.44127412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:52:55.237827708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:52:57.318694707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:52:57.328673087Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:52:57.796151741Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-04T12:52:59.441317844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:53:01.215127836Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-04T12:53:01.215149313Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=5
level=info ts=2026-05-04T12:53:03.816027641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:53:03.823027551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:53:04.442120028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:53:04.632161019Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T12:53:05.333254871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=debug ts=2026-05-04T12:53:05.333282563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-04T12:53:05.336827545Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T12:53:05.634203888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:53:05.637757381Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T12:53:08.049286547Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-04T12:53:08.663042431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:53:08.66509767Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T12:53:09.441289357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:10.746995999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:53:10.750509738Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T12:53:11.481399077Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T12:53:11.481423454Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T12:53:11.778602983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:53:11.783905849Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:53:12.112965311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:53:12.118341625Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:53:12.333957362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:53:12.338639131Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-04T12:53:14.442281453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:53:14.885015522Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T12:53:17.496796106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:53:17.500849508Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T12:53:18.303005783Z caller=checkpoint.go:372 msg="writing series" size="813 B"
level=info ts=2026-05-04T12:53:19.441268275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:53:19.441300381Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T12:53:19.441295405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:19.441313676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T12:53:19.937974502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:53:19.941820917Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T12:53:21.720892779Z caller=checkpoint.go:372 msg="writing series" size="391 kB"
level=info ts=2026-05-04T12:53:23.282327567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:53:23.289054606Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:53:23.673822408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T12:53:23.676447326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:53:23.677762128Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:53:23.679664699Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T12:53:24.441788106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:53:25.138330174Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
ts=2026-05-04T12:53:25.239106342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:53:27.619747932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:53:27.628551198Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:53:27.628576703Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:53:27.628606953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:53:27.635619548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:53:28.566086806Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-04T12:53:28.566118405Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=5
level=warn ts=2026-05-04T12:53:29.441317244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:29.824141137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:53:29.828030481Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T12:53:30.018597572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:53:30.024110154Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T12:53:31.973047219Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-04T12:53:34.44148043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:53:35.391275565Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T12:53:37.770049584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:53:37.774929559Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:53:38.80889865Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=debug ts=2026-05-04T12:53:38.829658985Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:53:38.829698056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:53:38.833849778Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T12:53:39.4412833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:39.608967131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:53:39.617548329Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:53:41.821807087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:53:41.825767753Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T12:53:42.226086841Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-04T12:53:44.442021202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:44.694216725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:53:44.704103907Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:53:45.644171107Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-04T12:53:47.737229701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:42654"
level=debug ts=2026-05-04T12:53:49.062079146Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T12:53:49.423151678Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:53:49.425265839Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:53:49.441573677Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:53:49.441610238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:53:49.441618669Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:53:49.441650062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:50.394805756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:53:50.399266995Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T12:53:50.685229507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:53:50.689419514Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T12:53:52.479535532Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-04T12:53:54.16606833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:58590"
level=warn ts=2026-05-04T12:53:54.441834659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:53:55.172718283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:53:55.177305454Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-04T12:53:55.240867608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:53:55.897810412Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-04T12:53:55.996844722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:53:56.002788894Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:53:56.565136871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:53:59.182426093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:53:59.187694582Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:53:59.254073534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:53:59.259626043Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:53:59.314787022Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-04T12:53:59.314809926Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-04T12:53:59.441280231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:54:01.440876328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:54:01.446651882Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:54:02.733590585Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=warn ts=2026-05-04T12:54:04.441945485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:54:06.150708117Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T12:54:06.398308824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:54:06.405597893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:54:08.403514858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:54:08.410098246Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:54:08.410125883Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:54:08.410155545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:54:08.415854928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:54:09.44197816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:54:09.56870796Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-04T12:54:12.847730579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:54:12.986280051Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=debug ts=2026-05-04T12:54:12.986303164Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T12:54:13.370087072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:54:13.374743383Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T12:54:14.442247447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:54:16.405633796Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-04T12:54:16.405657407Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
ts=2026-05-04T12:54:16.601480163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:32792"
level=info ts=2026-05-04T12:54:17.727234679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:54:17.730834316Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T12:54:19.441826392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:54:19.441858343Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:54:19.441866419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:54:19.441872438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:54:19.821894663Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T12:54:21.021625403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:54:21.027805128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:54:21.125026935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:54:21.128594733Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T12:54:23.241129571Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-04T12:54:23.241152117Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=info ts=2026-05-04T12:54:23.850406964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:54:23.854438993Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-04T12:54:24.16930936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46022"
level=warn ts=2026-05-04T12:54:24.441759199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:54:24.572097763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:54:24.580564024Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T12:54:25.242507546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:54:26.657425988Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-04T12:54:27.487002113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:54:27.493379501Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T12:54:29.441605125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:54:29.866629778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:54:29.875288177Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T12:54:30.074862221Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=debug ts=2026-05-04T12:54:33.492328986Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-04T12:54:34.441951089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:54:34.939274886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:54:34.945070953Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:54:35.032908695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:54:35.036870097Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T12:54:36.910225497Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-04T12:54:37.147224633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:54:39.441217398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:54:40.328221363Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:54:41.805272708Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:54:41.989860538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:54:41.995710532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:54:43.745841016Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-04T12:54:44.441986949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:54:46.603115815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:46704"
level=debug ts=2026-05-04T12:54:47.163304602Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-04T12:54:47.598601674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:54:47.60545603Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-04T12:54:47.741808315Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:41342"
level=debug ts=2026-05-04T12:54:49.42373077Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:54:49.424803864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:54:49.4411475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:54:49.441172687Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:54:49.441180531Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:54:49.441176513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:54:50.580935486Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-04T12:54:50.631295051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="736 kB" avg_comp="736 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-04T12:54:50.636584993Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19df2d52b82%3A19df2faccc0%3A9eb123ce?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-04T12:54:50.636613365Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:54:50.636636245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:54:50.63985096Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T12:54:54.001159928Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T12:54:54.001207368Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=9
level=warn ts=2026-05-04T12:54:54.441892516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:54:55.243708955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:54:56.726591864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:54:56.733112941Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:54:56.733138135Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:54:56.73316469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:54:56.738420534Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:54:57.416799333Z caller=checkpoint.go:372 msg="writing series" size="566 kB"
level=info ts=2026-05-04T12:54:57.83686975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:54:57.840956912Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:54:58.112613161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:54:58.118708824Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:54:58.642314078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:54:58.650572584Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:54:59.441264679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:00.835027899Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T12:55:00.835050827Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-04T12:55:01.357789606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:55:01.363725701Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T12:55:02.237466738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:55:03.36553375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:55:03.369563852Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T12:55:04.251783139Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=info ts=2026-05-04T12:55:04.342970968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:55:04.346913081Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T12:55:04.441316843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:07.669401564Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-04T12:55:09.402526393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:55:09.408830594Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T12:55:09.442221622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:11.087812892Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=info ts=2026-05-04T12:55:12.16995664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:55:12.174296648Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T12:55:12.214301852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:55:12.218032221Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T12:55:14.441397272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:55:14.473555639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:55:14.477579833Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T12:55:14.504842864Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-04T12:55:14.504867777Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
ts=2026-05-04T12:55:16.604653178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:34158"
level=debug ts=2026-05-04T12:55:17.923540876Z caller=checkpoint.go:372 msg="writing series" size="717 kB"
level=info ts=2026-05-04T12:55:19.441858371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:55:19.441885407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:55:19.441893791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:55:19.441920567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:21.341084374Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-04T12:55:21.432521419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:55:21.436134815Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T12:55:21.558407166Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T12:55:22.453117919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:55:22.461229662Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T12:55:24.176178762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:54346"
level=warn ts=2026-05-04T12:55:24.442096332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:24.758528015Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=debug ts=2026-05-04T12:55:24.75855061Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-04T12:55:25.2458028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
ts=2026-05-04T12:55:25.245956122Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=error ts=2026-05-04T12:55:25.24908798Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-04T12:55:26.491535333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:55:26.496550327Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T12:55:28.176297868Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-04T12:55:29.441426483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:29.717814495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:55:30.491960551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:55:30.49620698Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:55:31.593607868Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T12:55:33.376612656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:55:33.38447141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:55:33.384495621Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:55:33.38452125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:55:33.389678267Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T12:55:34.441600986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:55:35.011670558Z caller=checkpoint.go:372 msg="writing series" size="392 kB"
level=debug ts=2026-05-04T12:55:38.429444713Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T12:55:39.441538687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:55:39.593896062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:55:39.600117549Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:55:41.893291532Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-04T12:55:41.893330792Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=5
level=warn ts=2026-05-04T12:55:44.441508151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:55:44.702430693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:55:44.709229778Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:55:45.264788995Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=info ts=2026-05-04T12:55:45.412944547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:55:45.420422505Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:55:45.420449102Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:55:45.420476062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:55:45.427458051Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-04T12:55:47.746782805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:47422"
level=debug ts=2026-05-04T12:55:48.682623783Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-04T12:55:49.423548041Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:55:49.424601542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:55:49.441976265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:55:49.44200382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:55:49.442012012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:55:49.442027834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:55:51.508556481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:55:51.517172679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T12:55:51.540239316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:55:51.543858567Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T12:55:52.100396459Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T12:55:52.255006026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:55:52.258965763Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T12:55:54.441641826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:55:55.248548516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:55:55.51831597Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T12:55:55.518337664Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T12:55:55.883223357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:55:55.889879902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T12:55:56.323233842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:55:56.327207291Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T12:55:58.525031738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:55:58.529111342Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T12:55:58.935315087Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T12:55:59.441576115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:56:02.353610829Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-04T12:56:02.365698005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:56:02.369870293Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T12:56:04.442030731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:04.735771117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:56:04.744581048Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:56:05.668346118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:56:05.668422605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="134 kB" avg_uncomp="134 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T12:56:05.672484006Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T12:56:05.770762431Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-04T12:56:09.188820158Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-04T12:56:09.441887857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:09.940122308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:56:09.948771214Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:56:09.948795622Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:56:09.948822958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:56:09.956658056Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:56:12.262468736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:56:12.266312841Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T12:56:12.606377671Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-04T12:56:13.44431535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:56:13.448049529Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T12:56:14.441218391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:56:16.024709307Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T12:56:16.024731827Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T12:56:16.500790059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T12:56:16.504880208Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T12:56:17.939833508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:56:17.945504114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:56:19.442113952Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:56:19.44214808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:56:19.442158247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:56:19.442174482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:19.449127285Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-04T12:56:19.450379893Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009176382s
level=info ts=2026-05-04T12:56:20.359239386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:56:20.364774273Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:56:20.381442193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T12:56:24.441737066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:24.870450022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:56:24.877706511Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:56:25.148480227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:56:25.152034517Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-04T12:56:25.249675458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T12:56:26.182226479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:56:26.187450973Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T12:56:26.421382503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:56:26.428446872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:56:27.04815244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:56:27.058213282Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T12:56:29.441246313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:56:32.793891202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:56:34.441753793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-04T12:56:39.441964668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:43.716215548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:56:43.725241502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:56:44.441981154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:44.497788843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:56:44.50175528Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T12:56:44.753205195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:56:44.757355623Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-04T12:56:46.60967375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:58978"
level=info ts=2026-05-04T12:56:46.883237645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:56:46.887470985Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:56:47.562121276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:56:47.565842602Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-04T12:56:47.751816912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:55908"
level=info ts=2026-05-04T12:56:48.562120714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:56:48.568495623Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:56:49.423419129Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:56:49.42478734Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:56:49.441094157Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T12:56:49.441145587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:56:49.441167377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:56:49.441175513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:56:49.44121666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:56:49.44125447Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-04T12:56:49.441262155Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T12:56:49.441280384Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.61µs success=true
level=debug ts=2026-05-04T12:56:49.441505651Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-04T12:56:52.860130331Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T12:56:54.163932241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:56:54.16753181Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-04T12:56:54.185482303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:50862"
level=info ts=2026-05-04T12:56:54.280515066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T12:56:54.284321589Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T12:56:54.441851218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:56:55.250976269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T12:56:56.277583164Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T12:56:57.212316344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:56:57.220587872Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T12:56:57.229812023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:56:57.70511259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:56:57.70856552Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T12:56:59.441601783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:56:59.695143679Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-04T12:57:01.17476765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:57:01.179161047Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:57:01.179205975Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:57:01.179231626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:57:01.183102501Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:57:03.112488272Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-04T12:57:04.441997693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:57:06.531288791Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-04T12:57:09.441613872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:57:09.622995818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:57:09.629567183Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T12:57:09.948097647Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T12:57:10.098416406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:57:10.105783752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T12:57:11.908659981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:57:11.913275871Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T12:57:12.335233986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:57:12.341872585Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:57:13.365818452Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-04T12:57:14.441512546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:57:16.78408861Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-04T12:57:17.344750394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:57:17.349756788Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:57:19.44178138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:57:19.441810771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:57:19.441818713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:57:19.441826827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:57:19.544944843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T12:57:19.546708279Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T12:57:20.201861772Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-04T12:57:23.511108156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:57:23.515058803Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T12:57:23.619301408Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=info ts=2026-05-04T12:57:24.019448091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:57:24.02543359Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
ts=2026-05-04T12:57:24.189019603Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:60452"
level=warn ts=2026-05-04T12:57:24.442017161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:57:25.251967486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:57:27.037374513Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T12:57:27.037396542Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=warn ts=2026-05-04T12:57:29.441361126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:57:29.650298709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:57:30.455507161Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-04T12:57:30.455529279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
level=info ts=2026-05-04T12:57:33.073230778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:57:33.077275805Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T12:57:33.287808372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:57:33.295582688Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T12:57:33.872915006Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=warn ts=2026-05-04T12:57:34.441983882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:57:37.290012808Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T12:57:37.801482493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="544 B" avg_comp="544 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:57:37.805416772Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:57:39.352790592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T12:57:39.356709499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T12:57:39.442081907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:57:39.763467591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:57:39.774349523Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:57:40.707687887Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=info ts=2026-05-04T12:57:41.200003934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:57:41.204226228Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T12:57:43.01948944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:57:43.023318261Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T12:57:44.126084296Z caller=checkpoint.go:372 msg="writing series" size="951 kB"
level=debug ts=2026-05-04T12:57:44.126105994Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-04T12:57:44.442216694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:57:47.543287204Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-04T12:57:49.423500161Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:57:49.424553324Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:57:49.44191382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:57:49.441937331Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:57:49.441945202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:57:49.441958457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:57:50.316094385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:57:50.322590662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:57:50.385674344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:57:50.392996819Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:57:50.961450225Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=info ts=2026-05-04T12:57:51.631155155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:57:51.635635879Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T12:57:51.837238829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:57:51.841068579Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T12:57:54.378805807Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T12:57:54.442270683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:57:55.253859915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:57:55.818404252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:57:55.826635448Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:57:56.377933618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:57:56.383432874Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T12:57:56.594113598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:57:56.598092605Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T12:57:57.796740712Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-04T12:57:59.441700277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:00.4011405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:58:01.216993641Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-04T12:58:01.217014974Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=5
level=warn ts=2026-05-04T12:58:04.441912998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:04.509924353Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:58:04.631851357Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T12:58:07.87163238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:58:07.877382792Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T12:58:08.050000372Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-04T12:58:09.442212079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:11.478434085Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T12:58:11.478456371Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T12:58:12.105562373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:58:12.112094123Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T12:58:14.441336088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:14.885579675Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T12:58:15.397977825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:58:15.405898422Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T12:58:16.613724964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:47910"
ts=2026-05-04T12:58:17.757662511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:52498"
level=info ts=2026-05-04T12:58:18.077258935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:58:18.08103797Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T12:58:18.302601886Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-04T12:58:18.834062008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:58:18.837993559Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T12:58:19.441523501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:58:19.441551675Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:58:19.441559709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:58:19.441575457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:58:20.083205091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:58:20.089791245Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T12:58:21.721159428Z caller=checkpoint.go:372 msg="writing series" size="505 kB"
level=warn ts=2026-05-04T12:58:24.441880782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:25.13850319Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
ts=2026-05-04T12:58:25.256346931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:58:27.22170403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T12:58:27.226409007Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T12:58:28.562953997Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-04T12:58:28.562977162Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=5
level=warn ts=2026-05-04T12:58:29.441766613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:30.40644314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T12:58:31.973968634Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-04T12:58:32.953758148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:58:32.959342688Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:58:33.770306698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:58:33.774440774Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T12:58:34.441573494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:35.391087463Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T12:58:35.794539786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:58:35.800397627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:58:36.716236386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:58:36.720796723Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T12:58:37.066749843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:58:37.070796174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T12:58:38.809099062Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T12:58:39.441852225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:58:39.491957467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="399 B" avg_comp="399 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:58:39.495902708Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T12:58:40.444756085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:58:40.448632844Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T12:58:41.823822325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:58:41.831683411Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:58:42.155544492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:58:42.16175455Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:58:42.227164195Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-04T12:58:43.612981675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:58:43.619281144Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T12:58:44.441575474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:45.644586753Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-04T12:58:47.758678319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:33466"
level=debug ts=2026-05-04T12:58:49.062025726Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T12:58:49.423846253Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:58:49.424981521Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:58:49.441243707Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:58:49.441266974Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:58:49.441275102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:58:49.441280716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:51.776495672Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:58:52.480376449Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-04T12:58:54.199228505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42510"
level=warn ts=2026-05-04T12:58:54.441678155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:58:54.851949249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:58:54.859260008Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:58:54.859287183Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:58:54.859314179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:58:54.864460188Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
ts=2026-05-04T12:58:55.258284005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T12:58:55.898279841Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=debug ts=2026-05-04T12:58:59.315732449Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-04T12:58:59.441408628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:58:59.596167588Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T12:58:59.981481997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:58:59.98529456Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:58:59.985318107Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:58:59.985343938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:58:59.988326928Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T12:59:01.441738041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:59:01.445222996Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T12:59:02.733265717Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=debug ts=2026-05-04T12:59:02.733288767Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-04T12:59:03.694759885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:59:03.698997679Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T12:59:04.44207195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:59:06.150596645Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-04T12:59:09.367881802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T12:59:09.3741322Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T12:59:09.441580152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:59:09.569121624Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-04T12:59:12.585757895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:59:12.590104108Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T12:59:12.986706925Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=info ts=2026-05-04T12:59:13.633095599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:59:13.638501035Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T12:59:14.339240321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T12:59:14.346597676Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-04T12:59:14.442042667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:59:16.405583798Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-04T12:59:16.405605995Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=4
level=info ts=2026-05-04T12:59:17.620699613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T12:59:17.62464413Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T12:59:19.44180557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T12:59:19.441819766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:59:19.441838005Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:59:19.441847728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T12:59:19.821290873Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T12:59:20.318805442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T12:59:20.324611659Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T12:59:21.160957933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:59:21.165021563Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:59:21.165049946Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:59:21.165083684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:59:21.168823494Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T12:59:22.769510964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T12:59:22.77769777Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T12:59:23.02957229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:59:23.033351292Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:59:23.033373202Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:59:23.033395703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:59:23.036436487Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T12:59:23.241324997Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-04T12:59:23.2413472Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
ts=2026-05-04T12:59:24.202316421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:55894"
level=warn ts=2026-05-04T12:59:24.441916539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:59:24.88986626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T12:59:24.89384244Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T12:59:25.25999546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T12:59:25.941646313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:59:25.945139937Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:59:25.945162887Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:59:25.945200114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:59:25.94817244Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T12:59:26.265207025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T12:59:26.268590781Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T12:59:26.656881786Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=debug ts=2026-05-04T12:59:29.047778073Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T12:59:29.44175576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:59:30.07536401Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-04T12:59:31.337486133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:59:31.342075785Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:59:31.342100203Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:59:31.342123896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:59:31.345205099Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T12:59:31.98017889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T12:59:31.986704969Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T12:59:33.492846266Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-05-04T12:59:33.799233924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:59:33.804493032Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T12:59:33.918405527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:59:33.922016261Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T12:59:34.441962011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:59:35.341371465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T12:59:35.347482598Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T12:59:36.910057427Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=info ts=2026-05-04T12:59:37.458811549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T12:59:37.462210088Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T12:59:39.442126867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T12:59:40.328454894Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-04T12:59:43.74571564Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-04T12:59:44.050824534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T12:59:44.055314741Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T12:59:44.441197722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:59:44.594506118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T12:59:44.59855541Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-04T12:59:46.61984608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:57498"
level=debug ts=2026-05-04T12:59:47.163497148Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-04T12:59:47.579640579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T12:59:47.585376475Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T12:59:49.265452571Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T12:59:49.424050265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T12:59:49.425095249Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T12:59:49.441348414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T12:59:49.441382194Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T12:59:49.441390001Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T12:59:49.441402241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T12:59:50.088924921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T12:59:50.092381245Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T12:59:50.581169761Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-04T12:59:50.943150606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:59:50.946651002Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-04T12:59:50.946675566Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T12:59:50.946701506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T12:59:50.949816473Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T12:59:54.000060726Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-04T12:59:54.000088558Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=9
level=info ts=2026-05-04T12:59:54.342554313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T12:59:54.346641984Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T12:59:54.441907371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T12:59:55.262276996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T12:59:56.405044398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T12:59:56.409368477Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T12:59:57.416927235Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=warn ts=2026-05-04T12:59:59.44208718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:00.834814862Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T13:00:00.834836888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=debug ts=2026-05-04T13:00:02.082058782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:00:04.252077297Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-04T13:00:04.441565782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:00:05.086970772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:00:05.09512918Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:00:06.13292575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:00:06.140559636Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:00:06.140585386Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:00:06.14061561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:00:06.148074064Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T13:00:07.613251263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:00:07.620874312Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T13:00:07.670033747Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-04T13:00:08.783606157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:00:08.791921497Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T13:00:09.441393491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:00:09.872321202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:00:09.876977171Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:00:10.579093844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:00:10.583511675Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:00:11.088128035Z caller=checkpoint.go:372 msg="writing series" size="456 kB"
level=info ts=2026-05-04T13:00:13.216238678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:00:13.222024042Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T13:00:14.441228712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:14.505241656Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-04T13:00:14.50526384Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
ts=2026-05-04T13:00:17.763604293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:60076"
level=debug ts=2026-05-04T13:00:17.922785869Z caller=checkpoint.go:372 msg="writing series" size="730 kB"
level=warn ts=2026-05-04T13:00:19.441816037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:00:19.44181517Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:00:19.441844404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:00:19.441853996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T13:00:20.433827527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:00:20.438514992Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:00:20.43853961Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:00:20.438566929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:00:20.441773737Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:00:20.503890554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:00:20.509681975Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:00:21.341284439Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-04T13:00:22.365198314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:00:22.370132243Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:00:23.304763031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:00:23.308676643Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T13:00:24.441400251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:24.758402699Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-04T13:00:24.758424432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-04T13:00:25.088415621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:00:25.095697083Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T13:00:25.264419076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T13:00:27.52540422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:00:27.529315504Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T13:00:28.176039622Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-04T13:00:28.638616926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:00:28.642675615Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T13:00:29.226669503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T13:00:29.441613159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:31.593495176Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T13:00:34.276070366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:00:34.280037671Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T13:00:34.441542011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:35.011595488Z caller=checkpoint.go:372 msg="writing series" size="483 kB"
level=info ts=2026-05-04T13:00:37.870907113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:00:37.876466838Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:00:38.429419108Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T13:00:39.442197066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:00:39.734091573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:00:39.738617531Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:00:39.914309475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:00:39.918583704Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-04T13:00:40.785815126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:00:40.789804546Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:00:41.890794458Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-04T13:00:41.890824694Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=5
level=info ts=2026-05-04T13:00:43.954269266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:00:43.959784918Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T13:00:44.441874159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:45.265493345Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=debug ts=2026-05-04T13:00:45.708297351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-04T13:00:47.764672716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:36542"
level=debug ts=2026-05-04T13:00:48.683342311Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-04T13:00:49.423243032Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:00:49.424286964Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-04T13:00:49.425715939Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-04T13:00:49.42577465Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975443 n=1
level=debug ts=2026-05-04T13:00:49.425835476Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975443 n=1
level=info ts=2026-05-04T13:00:49.441674247Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:00:49.441699047Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:00:49.441709039Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:00:49.441726531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:00:52.100703074Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T13:00:53.120602929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:00:53.127690258Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:00:53.127724833Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:00:53.127750475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:00:53.133905528Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-04T13:00:54.212335674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46380"
level=warn ts=2026-05-04T13:00:54.441886652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:00:55.266210004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:00:55.517952464Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T13:00:55.517975824Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T13:00:58.052081446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:00:58.061341509Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T13:00:58.936029777Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T13:00:59.441714297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:01.282239274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:01:01.286040355Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:01:01.507731876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:01:01.513423204Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:01:01.513448187Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:01:01.513472803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:01:01.519158665Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:01:01.806898364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:01:01.816274984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:01:02.353679353Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-04T13:01:02.756968437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:01:02.762173477Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:01:04.441444652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:04.861604656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:01:04.865980101Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:01:05.046148931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:01:05.052300737Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:01:05.062424152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:01:05.062437919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="201 B" avg_comp="201 B" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:01:05.065618157Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T13:01:05.771460009Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-04T13:01:08.232615305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:01:08.237433118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:01:09.189509657Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-04T13:01:09.441412726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:11.258764338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:01:11.264603561Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:01:12.606917457Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-04T13:01:14.44209542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:01:16.024652114Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T13:01:16.024674724Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T13:01:16.970094691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:01:16.975296084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T13:01:17.583977362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:01:17.588315757Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-04T13:01:17.76623403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:57532"
level=info ts=2026-05-04T13:01:17.920384331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:01:17.92454954Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:01:18.507926015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:01:18.513712143Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:01:19.44127479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:01:19.441314371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:01:19.441322364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:01:19.44131916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:19.450013843Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-04T13:01:19.451379793Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009956752s
level=info ts=2026-05-04T13:01:20.648554024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:01:20.652561812Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:01:24.155769444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:01:24.159837462Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:01:24.441779011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:01:25.26856692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=warn ts=2026-05-04T13:01:29.441933478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:29.693243893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:01:29.699571338Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:01:31.389649468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:01:31.393690229Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:01:33.727875009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:01:33.731428162Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T13:01:34.441974461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:38.120606173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:01:38.124475028Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-04T13:01:38.124499869Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:01:38.12452518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:01:38.127599577Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T13:01:39.115100543Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T13:01:39.4422312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:01:43.087168357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:01:43.089109031Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-04T13:01:44.442243434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:44.988105925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:01:44.993618561Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:01:45.669022002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:01:45.675024678Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:01:45.675047477Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:01:45.675074932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:01:45.6804773Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:01:46.676315865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:01:46.683302067Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-04T13:01:47.767305494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:49094"
level=info ts=2026-05-04T13:01:47.91839017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:01:47.926785446Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T13:01:48.934471385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:01:48.937570203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:01:49.42330995Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:01:49.424375613Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:01:49.441782575Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T13:01:49.441781767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:01:49.441806124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:01:49.441817124Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:01:49.44181491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:01:49.441932368Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-04T13:01:49.441942519Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T13:01:49.44196429Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.872µs success=true
level=debug ts=2026-05-04T13:01:49.442251834Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-04T13:01:49.719373688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:01:49.725028775Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:01:51.332709497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:01:51.336237362Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-04T13:01:51.544031019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:01:51.549257891Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:01:51.549280878Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:01:51.549305098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:01:51.553938147Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T13:01:52.860682741Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T13:01:54.333151147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:01:54.337141661Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:01:54.441504621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:01:55.271807108Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:01:55.353201898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:01:55.35699479Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T13:01:56.278591312Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-04T13:01:56.329859896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:01:56.336515661Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:01:58.38954929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:01:58.393723193Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T13:01:59.441991239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:01:59.696056815Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T13:02:03.113827876Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-04T13:02:04.441416764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:02:04.471564516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:02:04.475397664Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:02:04.644162046Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:02:06.532033261Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-05-04T13:02:06.636303452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:02:06.640119131Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:02:07.768166957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:02:07.771917897Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T13:02:09.442220823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:02:09.94959956Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T13:02:11.285271427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:02:11.295164835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:02:12.591142765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:02:12.594823897Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T13:02:13.367067198Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-04T13:02:14.441593878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:02:16.784624296Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-04T13:02:19.441174145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T13:02:19.441202999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:02:19.441217904Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:02:19.441228709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T13:02:20.203027167Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-05-04T13:02:20.203049366Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-04T13:02:20.918155013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:02:20.922047994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T13:02:23.619504364Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-04T13:02:23.660802074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:02:24.081799646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:02:24.08551506Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-04T13:02:24.221075241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:42908"
level=warn ts=2026-05-04T13:02:24.442131113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:02:25.274326195Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=info ts=2026-05-04T13:02:26.966449976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="544 B" avg_comp="544 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:02:26.970263891Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T13:02:27.03786496Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=info ts=2026-05-04T13:02:27.751762947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:02:27.758925088Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:02:28.296378191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:02:28.30211958Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T13:02:29.441525273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:02:30.457181881Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-04T13:02:30.457225172Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=debug ts=2026-05-04T13:02:33.795569883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:02:33.873071688Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-04T13:02:34.441571014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:02:35.002754424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:02:35.009919292Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:02:35.450529744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:02:35.45692928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:02:36.098666964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:02:36.102498912Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:02:37.290951175Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-04T13:02:39.442160042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:02:40.709120983Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-04T13:02:41.125235599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:02:41.129368771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T13:02:42.890907555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:02:42.90080464Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:02:42.900829682Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:02:42.900860969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:02:42.907691004Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T13:02:44.12807697Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T13:02:44.128099635Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-05-04T13:02:44.44181027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:02:45.356029023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:02:45.359559582Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:02:45.35958014Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:02:45.35960013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:02:45.362398329Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:02:45.705602766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:02:45.71314293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:02:47.544094731Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-04T13:02:49.42337921Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:02:49.424413866Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-04T13:02:49.441835457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:02:49.441838526Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:02:49.441863711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:02:49.441871284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T13:02:49.609774656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:02:49.614679287Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:02:49.635302519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:02:49.641409705Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:02:50.962265602Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-04T13:02:54.089426713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:02:54.093734729Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-04T13:02:54.22457176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:36068"
level=debug ts=2026-05-04T13:02:54.379265787Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T13:02:54.441409804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:02:55.037394965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:02:55.045030718Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T13:02:55.277096293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:02:57.797226084Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-04T13:02:59.441813095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:02:59.629909433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:02:59.634173038Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:03:01.217810958Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-04T13:03:01.217835704Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=5
level=debug ts=2026-05-04T13:03:02.267512667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:03:02.449474306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:03:02.453580695Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T13:03:04.441398991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:04.630795186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T13:03:04.632115421Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=error ts=2026-05-04T13:03:04.634757972Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T13:03:08.050444074Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T13:03:08.119692676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:03:08.123805478Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T13:03:09.44215826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:09.698147637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:03:09.705068658Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:03:10.158693218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:03:10.162129505Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T13:03:11.478661637Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T13:03:11.478684294Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T13:03:12.736790848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:03:12.740454152Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T13:03:13.307984071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:03:13.313421794Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T13:03:14.441970189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:03:14.885628773Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T13:03:16.944773998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:03:16.948562103Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:03:17.262912656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:03:17.269787199Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:03:17.60170801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:03:17.609907835Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T13:03:17.773691213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:53990"
level=debug ts=2026-05-04T13:03:18.303508036Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-04T13:03:19.441753752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:03:19.441782768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:03:19.441791082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:03:19.441798745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:21.190406907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:03:21.194590536Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T13:03:21.721706635Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=info ts=2026-05-04T13:03:23.413707837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:03:23.417794025Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-04T13:03:24.227808167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:48420"
level=debug ts=2026-05-04T13:03:24.24395318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T13:03:24.441547577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:03:25.138981116Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
ts=2026-05-04T13:03:25.279836266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T13:03:26.901256868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:03:28.562383726Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-04T13:03:28.562407588Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=5
level=warn ts=2026-05-04T13:03:29.441362428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:03:31.974163856Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-04T13:03:33.550180921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:03:33.555461734Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:03:33.62181905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:03:33.62746787Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T13:03:34.441585993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:34.846198239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:03:34.85037043Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:03:34.850396369Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:03:34.850421113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:03:34.85390522Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:03:35.392331662Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-04T13:03:38.810461679Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T13:03:39.442043687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:40.968389764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:03:40.972317116Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:03:41.129909864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:03:41.136941594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:03:42.227356215Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-04T13:03:43.696561957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:03:43.702169194Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:03:44.442099671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:03:45.64521087Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-04T13:03:49.062511263Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T13:03:49.423263622Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:03:49.424378138Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:03:49.441732185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:03:49.441759646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:03:49.441768477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:03:49.441763605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:51.055450528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:03:51.058794828Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T13:03:51.875113752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:03:51.878779575Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:03:52.480626875Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T13:03:54.339613307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:03:54.343759201Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T13:03:54.44219862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:54.607514973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:03:54.61172419Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T13:03:55.077471509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T13:03:55.282268296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:03:55.477546328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:03:55.483288953Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:03:55.898834305Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-04T13:03:56.508541144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:03:56.514801939Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:03:59.316631322Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-04T13:03:59.441985013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:03:59.798314443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:03:59.802486872Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T13:04:02.73433064Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-04T13:04:02.734354731Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-04T13:04:03.366593622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:04:03.370592361Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T13:04:04.4416085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:04:06.151131958Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-04T13:04:09.441269568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:04:09.570082126Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=debug ts=2026-05-04T13:04:09.585920196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:04:10.49331453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:04:10.498933804Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:04:12.987552153Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=warn ts=2026-05-04T13:04:14.441278746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:04:14.887699879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:04:14.894371026Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:04:14.992636449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:04:14.996329181Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:04:15.548852866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:04:15.556590139Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T13:04:16.406101499Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-04T13:04:16.40612326Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=info ts=2026-05-04T13:04:16.418809854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="399 B" avg_comp="399 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:04:16.422497727Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:04:19.441232818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:04:19.441262028Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:04:19.441270412Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:04:19.441264121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:04:19.822788761Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T13:04:20.409633781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:04:20.416217567Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:04:22.889554396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:04:22.893725609Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T13:04:23.167449665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:04:23.17133303Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T13:04:23.24174773Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-04T13:04:23.241771952Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=2
ts=2026-05-04T13:04:24.23507781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:56864"
level=warn ts=2026-05-04T13:04:24.441323755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:04:25.284355309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T13:04:26.264937838Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:04:26.65810105Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-04T13:04:28.529236143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:04:28.53391128Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:04:29.442001096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:04:29.896607189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:04:29.905437663Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:04:30.076025406Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-04T13:04:30.874316336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:04:30.880625959Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:04:31.188446646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:04:31.191901007Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T13:04:33.493844151Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-04T13:04:34.441833939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:04:35.101425864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:04:35.105956279Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:04:36.911051256Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-04T13:04:39.300430451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:04:39.304174169Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T13:04:39.441965709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:04:39.918472641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:04:39.925840446Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:04:40.328922985Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-04T13:04:41.120223156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:04:41.123911983Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T13:04:43.746859283Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-05-04T13:04:43.793905067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.8 MB" avg_uncomp="7.8 MB" synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:04:43.799173505Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T13:04:44.441685941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:04:47.164521607Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
ts=2026-05-04T13:04:47.781583223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:44550"
level=debug ts=2026-05-04T13:04:49.423496747Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:04:49.424548548Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:04:49.441934413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:04:49.441956963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:04:49.441965446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:04:49.441985337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:04:50.58225559Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=info ts=2026-05-04T13:04:52.161016745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:04:52.167031292Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:04:54.001178289Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-04T13:04:54.001219074Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=9
level=warn ts=2026-05-04T13:04:54.442001916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:04:55.285726193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:04:57.418085142Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
level=debug ts=2026-05-04T13:04:58.31095437Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:04:59.108676533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:04:59.112080479Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T13:04:59.442175217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:05:00.003347267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:05:00.835399719Z caller=checkpoint.go:372 msg="writing series" size="930 kB"
level=debug ts=2026-05-04T13:05:00.835425018Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=debug ts=2026-05-04T13:05:04.253287089Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=info ts=2026-05-04T13:05:04.359406583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:05:04.363470665Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T13:05:04.441820342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:05.536973573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:05:05.540878335Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:05:06.844649615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:05:06.848961373Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:05:07.670416056Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-04T13:05:08.404434153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:05:08.408263231Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:05:09.106256063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:05:09.11257178Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:05:09.441245817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:05:11.088487382Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=info ts=2026-05-04T13:05:11.558806464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:05:11.56311986Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T13:05:12.46501413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:05:12.468920486Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:05:12.468943544Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:05:12.468965577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:05:12.472227569Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T13:05:14.44193626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:05:14.505885888Z caller=checkpoint.go:372 msg="writing series" size="704 kB"
level=debug ts=2026-05-04T13:05:14.505909969Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-04T13:05:14.651064409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:05:14.658487428Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
ts=2026-05-04T13:05:16.645073554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:45718"
level=debug ts=2026-05-04T13:05:17.924169007Z caller=checkpoint.go:372 msg="writing series" size="705 kB"
level=info ts=2026-05-04T13:05:18.87364769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:05:18.877499517Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:05:19.442107909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:19.442114433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:05:19.442135793Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:05:19.442144049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T13:05:21.341666764Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
ts=2026-05-04T13:05:24.241882702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:60906"
level=warn ts=2026-05-04T13:05:24.441855967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:05:24.759555293Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=debug ts=2026-05-04T13:05:24.759578739Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-04T13:05:25.288418136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:05:25.791237657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:05:25.795377369Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T13:05:26.446513493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:05:26.454542453Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:05:27.470607386Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:05:27.470679116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="368 B" avg_comp="368 B" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" idle=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:05:27.47446177Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:05:28.176728048Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-04T13:05:28.492812868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:05:28.500664728Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T13:05:28.791431309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:05:28.795398615Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T13:05:29.441223094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:30.577237771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:05:30.583178989Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:05:30.583225341Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:05:30.583253062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:05:30.588878972Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:05:31.594705846Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T13:05:32.335850111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:05:32.339576362Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:05:33.612667211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:05:33.619407354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T13:05:34.441368054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:05:35.012436859Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=info ts=2026-05-04T13:05:36.707976999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:05:36.712033577Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-04T13:05:38.430238216Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-04T13:05:39.44128907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:41.864483304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:05:41.868476671Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:05:41.896284642Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-04T13:05:41.896316977Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=5
level=warn ts=2026-05-04T13:05:44.441913358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:44.454019442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:05:44.458085259Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:05:44.45811149Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:05:44.458136513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:05:44.461505406Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:05:45.266024175Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
level=info ts=2026-05-04T13:05:45.53876024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:05:45.545100891Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:05:47.488779502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:05:47.495878411Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:05:48.683722467Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=info ts=2026-05-04T13:05:49.369781956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:05:49.377205069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T13:05:49.423316264Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:05:49.424350033Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:05:49.441725985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:05:49.44175204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-04T13:05:49.441745545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:49.441759076Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T13:05:51.531236553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:05:51.535310202Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:05:51.535337611Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:05:51.535362425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:05:51.538652111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T13:05:52.101253962Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T13:05:53.100316576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:05:53.104378539Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-04T13:05:54.244735055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:39048"
level=warn ts=2026-05-04T13:05:54.441860101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:05:55.05191262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:05:55.056020131Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:05:55.056051423Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:05:55.056077192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:05:55.059447469Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-04T13:05:55.291381334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:05:55.518896723Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T13:05:55.518920763Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-04T13:05:55.716360242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:05:55.720005642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T13:05:55.988163797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:05:55.990284876Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T13:05:56.025748475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:05:56.030330668Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:05:56.808444081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:05:56.811992711Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:05:58.936420672Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T13:05:59.441995135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:06:02.353806239Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T13:06:03.246490295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:06:04.266398379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:06:04.270388474Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:06:04.441929096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:05.209578131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:06:05.213491912Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T13:06:05.77160672Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-04T13:06:07.720628149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:06:07.726361304Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:06:09.189730766Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-04T13:06:09.300016359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:06:09.306839729Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T13:06:09.44123982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:09.63530998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:06:09.639344474Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:06:10.801805228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:06:10.805913444Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T13:06:12.60750162Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-04T13:06:12.903435832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:06:12.911507416Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T13:06:14.441266573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:14.980168199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:06:14.98789216Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T13:06:15.785242575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:06:15.788995555Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:06:16.025268618Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T13:06:16.025291442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-04T13:06:17.785857679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:55890"
level=info ts=2026-05-04T13:06:18.172522748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:06:18.177691583Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T13:06:19.442033621Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:06:19.442062443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:06:19.442070693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:06:19.442116612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:19.450443169Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-04T13:06:19.451802279Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009646384s
level=info ts=2026-05-04T13:06:22.416173459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:06:22.419833142Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-04T13:06:22.419859947Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:06:22.419883271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:06:22.42304828Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T13:06:24.441340127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:06:25.294482901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:06:27.019042871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:06:27.023966575Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:06:29.206754704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:06:29.213316663Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T13:06:29.441445351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:29.953913738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:06:29.95788937Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:06:29.957913887Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:06:29.957936201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:06:29.960792263Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:06:31.817484773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:06:31.82440332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:06:33.411869163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:06:33.41560219Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-04T13:06:33.85764971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:06:33.861960689Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T13:06:34.441683729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:06:35.635340615Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:06:37.262137493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:06:37.266234466Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T13:06:39.442071859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:43.208304225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:06:43.211533213Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:06:44.179120868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:06:44.184628028Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T13:06:44.441542648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:06:44.709794701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:06:45.04101405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:06:45.044307479Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T13:06:45.613128557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:06:45.617102263Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:06:49.196227762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:06:49.200110775Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:06:49.276523823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:06:49.281682475Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:06:49.423236473Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:06:49.42430524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:06:49.441676215Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:06:49.441702019Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:06:49.44171051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T13:06:49.441699803Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-04T13:06:49.441711513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:06:49.441881744Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-04T13:06:49.441890271Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T13:06:49.441906936Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.48µs success=true
level=debug ts=2026-05-04T13:06:49.442139437Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-04T13:06:52.049934224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:06:52.055847848Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:06:52.860289969Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-04T13:06:53.231379724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:06:53.23518068Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:06:53.388446856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:06:53.392086458Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:06:53.392107614Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:06:53.39213038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:06:53.395239252Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:06:54.44214409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:06:55.297169529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:06:56.277881722Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-04T13:06:59.442032401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:06:59.696059871Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-04T13:07:00.908768808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:07:00.920687553Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=debug ts=2026-05-04T13:07:01.611847619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:07:03.113462714Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-04T13:07:04.441716364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:06.531491981Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-04T13:07:06.963491898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:07:06.967753828Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T13:07:08.519210324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:07:08.52316393Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:07:09.441393046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:09.949166935Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T13:07:10.166765622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:07:10.170723512Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:07:10.584087162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:07:10.592138561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:07:10.592162277Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:07:10.59222058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:07:10.599668165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T13:07:12.699362786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:07:12.703261887Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=info ts=2026-05-04T13:07:12.882829679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:07:12.886757164Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:07:13.367131156Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-04T13:07:14.441910366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:16.784271315Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-04T13:07:19.441804377Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:07:19.44184641Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:07:19.44185468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:07:19.441808353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:20.201793109Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-04T13:07:20.703398925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:07:20.712065534Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:07:22.43947875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:07:22.44627664Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:07:23.620005777Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
ts=2026-05-04T13:07:24.253451553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:52076"
level=warn ts=2026-05-04T13:07:24.441743614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:07:25.299566861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:07:25.315160484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:07:25.323367409Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:07:25.947360031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:07:27.038476046Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T13:07:27.038498362Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=info ts=2026-05-04T13:07:28.431445886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:07:28.437959999Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:07:29.44208092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:30.457941152Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-04T13:07:30.457963588Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
level=info ts=2026-05-04T13:07:30.469810832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:07:30.472971896Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T13:07:30.647545384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:07:30.65104898Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-04T13:07:31.158626514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:07:31.163775531Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:07:32.356648625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:07:32.512112944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:07:32.515716173Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:07:33.873069788Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-04T13:07:34.371626517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:07:34.377502649Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=warn ts=2026-05-04T13:07:34.44203801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:07:35.54575679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:07:35.552064083Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:07:35.55209456Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:07:35.552125838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:07:35.558094466Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:07:35.59821343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:07:35.601783682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:07:35.937839417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:07:35.941675804Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T13:07:37.290647191Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T13:07:38.557167665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:07:38.561086904Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:07:38.561115827Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:07:38.561144177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:07:38.564623129Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:07:39.441616396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:40.709025079Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=debug ts=2026-05-04T13:07:44.126546335Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T13:07:44.126569082Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-04T13:07:44.441523263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:07:47.208764282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:07:47.216935429Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:07:47.544109216Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-04T13:07:48.521003906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:07:48.525266633Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-04T13:07:48.570321837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:07:48.577286106Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:07:48.577311475Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:07:48.577336946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:07:48.582959034Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:07:49.423341645Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:07:49.424406461Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:07:49.441770795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:07:49.441797585Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:07:49.441805941Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:07:49.441802098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:07:50.073235333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:07:50.078772924Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:07:50.961577003Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-04T13:07:54.379812355Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T13:07:54.442153135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:07:54.741013495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:07:54.747238932Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-04T13:07:55.301677428Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T13:07:57.796915206Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-04T13:07:58.173953571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:07:58.182404433Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=info ts=2026-05-04T13:07:58.646451587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:07:58.653170092Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T13:07:59.442218685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:07:59.744761891Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:08:01.21824346Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-04T13:08:01.218268616Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=5
level=info ts=2026-05-04T13:08:02.503304322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="544 B" avg_comp="544 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:08:02.509322243Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T13:08:04.442126587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:08:04.632999135Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T13:08:06.454332935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:08:06.461364508Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:08:06.461394375Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:08:06.461424829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:08:06.468024635Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:08:08.050644669Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-04T13:08:09.151311953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:08:09.155876575Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=warn ts=2026-05-04T13:08:09.441992967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:08:10.952778031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:08:10.956754094Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:08:11.479308011Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T13:08:11.479331305Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T13:08:14.410235762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:08:14.414682224Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:08:14.441667366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:08:14.885492603Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T13:08:15.221288629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:08:15.224952954Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:08:16.681381716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:08:16.685342121Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:08:17.410501508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:08:17.414465162Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:08:17.506843973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:08:17.512795452Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
ts=2026-05-04T13:08:17.795029719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43938"
level=info ts=2026-05-04T13:08:18.095597981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:08:18.100340176Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:08:18.302884144Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-04T13:08:18.545135027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:08:18.551221157Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:08:18.551250375Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:08:18.551276759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:08:18.556939727Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:08:18.579990309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:08:18.586124784Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:08:19.441272363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:08:19.44131153Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:08:19.441320197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:08:19.441369438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:08:21.458919413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-04T13:08:21.46261317Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19df2d4eaaa%3A19df2d4eaab%3A8ab3b06f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-04T13:08:21.462635179Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:08:21.462661616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:08:21.467127377Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-04T13:08:21.721159316Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=info ts=2026-05-04T13:08:22.783401977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:08:22.789443384Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-04T13:08:24.441871861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:08:25.139236329Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
ts=2026-05-04T13:08:25.303261822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T13:08:25.423587586Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:08:25.588373539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:08:25.828305969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:08:25.831836386Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T13:08:26.285042489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:08:26.291464842Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:08:28.566840544Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-04T13:08:28.566864763Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=5
level=warn ts=2026-05-04T13:08:29.441692927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:08:31.535832997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:08:31.541030648Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T13:08:31.974222407Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-04T13:08:33.28163491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:08:33.285822375Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:08:34.441451625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:08:35.391847523Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T13:08:37.937306946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:08:37.941658071Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:08:38.810294045Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-04T13:08:39.344399571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:08:39.349539336Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-04T13:08:39.441976723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:08:40.761611909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:08:40.768780674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:08:42.227228596Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-04T13:08:42.938378819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:08:42.946289208Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=warn ts=2026-05-04T13:08:44.441688537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:08:45.644945172Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-04T13:08:47.68339925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:08:47.689865853Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-04T13:08:47.796271807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:58518"
level=debug ts=2026-05-04T13:08:49.063214538Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T13:08:49.423636991Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:08:49.424679037Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:08:49.442056472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:08:49.442086189Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:08:49.44210427Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:08:49.442109987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:08:52.480908106Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-04T13:08:52.666642384Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:08:54.158511619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:08:54.162782747Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:08:54.375415515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:08:54.379717689Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T13:08:54.441974722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:08:55.305044469Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:08:55.898823889Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-04T13:08:56.63090641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:08:56.635574838Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:08:59.316433565Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-04T13:08:59.442147394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:00.470980719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:09:00.476310526Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:09:01.493976678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:09:01.764767515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:09:01.768541484Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:09:01.768564231Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:09:01.768588593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:09:01.771715746Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T13:09:02.733642981Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-04T13:09:02.733667102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-04T13:09:02.958110856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:09:02.963840394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:09:04.442007742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:04.524831532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="399 B" avg_comp="399 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:09:04.528692747Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:09:06.151057188Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-04T13:09:08.789172197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:09:08.79401095Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-04T13:09:09.442214611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:09:09.569433521Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-04T13:09:12.987324168Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=info ts=2026-05-04T13:09:13.54488835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:09:13.549647926Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:09:14.096758337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:09:14.100756159Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:09:14.144954806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:09:14.148932952Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-04T13:09:14.44192518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:15.311611905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:09:15.313872478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:09:15.315841012Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:09:15.315846971Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:09:15.828393937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:09:15.834061619Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:09:16.406131676Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-04T13:09:16.406157047Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=debug ts=2026-05-04T13:09:18.495355035Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:09:19.441814678Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:09:19.441844713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:09:19.441852701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:09:19.441870103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:09:19.822522065Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T13:09:20.666574659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:09:20.672686957Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:09:22.247174693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:09:22.254281008Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:09:23.241730348Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-04T13:09:23.241756084Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=warn ts=2026-05-04T13:09:24.442122772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:09:25.307380391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:09:26.657893743Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-04T13:09:28.497099284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:09:28.502312208Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:09:29.442070652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:09:30.075799529Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=debug ts=2026-05-04T13:09:33.493051122Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-04T13:09:34.441981783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:35.204242695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=01147c482cf78ef1 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:09:35.212563319Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1147c482cf78ef1/19df2df8a14%3A19df2eb78a5%3A9926fd45?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-wssv\"}"
level=error ts=2026-05-04T13:09:35.212588961Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:09:35.212614533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:09:35.215952049Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-04T13:09:36.91084343Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-04T13:09:37.917934117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:09:37.922634869Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T13:09:39.441790549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:09:40.328907747Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-04T13:09:43.567735118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:09:43.574427564Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:09:43.746437505Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-05-04T13:09:43.825335328Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T13:09:44.44138889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:45.200766849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:09:45.204617624Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:09:45.531832688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:09:45.537665636Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T13:09:47.164144931Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-04T13:09:47.563373403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:09:47.568956287Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:09:49.42350594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:09:49.424551264Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:09:49.441909186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:09:49.441935174Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:09:49.441944827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:09:49.441942117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:49.463229529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:09:49.467075031Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:09:50.581754374Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=info ts=2026-05-04T13:09:52.802435069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:09:52.806409775Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:09:53.522645731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:09:53.526369558Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:09:54.001276981Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-04T13:09:54.001301041Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=9
level=info ts=2026-05-04T13:09:54.08880083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:09:54.092640153Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-04T13:09:54.301234644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:09:54.305174472Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:09:54.44180867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:09:54.599456816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:09:54.603553944Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:09:54.862009963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:09:54.866404899Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:09:54.86642632Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:09:54.866451088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:09:54.870172784Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-04T13:09:55.310645934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:09:57.417129276Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=info ts=2026-05-04T13:09:59.036093944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:09:59.040052734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T13:09:59.441199412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:10:00.835730179Z caller=checkpoint.go:372 msg="writing series" size="927 kB"
level=debug ts=2026-05-04T13:10:00.835754199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-04T13:10:01.137851872Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:10:01.139832212Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-04T13:10:01.339401733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:10:01.344283338Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-04T13:10:02.310987098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:10:02.318347511Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T13:10:02.787617417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:10:02.794573236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:10:04.253307224Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-04T13:10:04.441384856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:10:04.473651441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:10:04.479667231Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T13:10:06.916792605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:10:06.921061808Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:10:07.670233793Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-04T13:10:09.441740107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:10:11.08794287Z caller=checkpoint.go:372 msg="writing series" size="509 kB"
level=info ts=2026-05-04T13:10:13.888935277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:10:13.894705143Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-04T13:10:14.441710625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:10:14.506259608Z caller=checkpoint.go:372 msg="writing series" size="705 kB"
level=debug ts=2026-05-04T13:10:14.506284214Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
ts=2026-05-04T13:10:16.669309718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:60504"
level=debug ts=2026-05-04T13:10:17.924559127Z caller=checkpoint.go:372 msg="writing series" size="720 kB"
level=debug ts=2026-05-04T13:10:18.807174079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:10:19.441741405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:10:19.441770685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:10:19.441778759Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:10:19.441806028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:10:20.091240679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:10:20.09694191Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T13:10:21.341526694Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-04T13:10:21.755797482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:10:21.759936084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T13:10:24.441315189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:10:24.759835743Z caller=checkpoint.go:372 msg="writing series" size="431 kB"
level=debug ts=2026-05-04T13:10:24.759858348Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-04T13:10:25.313089127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:10:28.177231649Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-04T13:10:29.215465994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:10:29.22073591Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-04T13:10:29.441360566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:10:30.246345504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:10:30.250651102Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:10:31.594583232Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-04T13:10:33.109988378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:10:33.114080499Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T13:10:33.799931496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:10:33.803857146Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:10:34.441559324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:10:35.012261954Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-04T13:10:36.742406227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:10:36.745783141Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:10:37.652098935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:10:37.657956452Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-04T13:10:38.430298901Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-04T13:10:38.580403252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:10:38.584179822Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:10:39.44126345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:10:39.500054183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:10:39.506339431Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:10:39.664890067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:10:39.668592157Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:10:41.896118144Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-04T13:10:41.896157045Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=5
level=info ts=2026-05-04T13:10:42.658843908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:10:42.665080304Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:10:44.15113708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:10:44.156370024Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-04T13:10:44.441474958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:10:45.265354466Z caller=checkpoint.go:372 msg="writing series" size="305 kB"
ts=2026-05-04T13:10:46.670510952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:49176"
level=info ts=2026-05-04T13:10:47.84977826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:10:47.856911408Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:10:48.684049711Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-04T13:10:49.423234814Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:10:49.42427758Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:10:49.441637618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:10:49.441667388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:10:49.441685767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:10:49.441714125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:10:49.676028264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:10:49.679857224Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T13:10:52.100688542Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-04T13:10:52.45306278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:10:52.457357262Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:10:52.55657693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.6 MB" total_uncomp="94 MB" avg_uncomp="23 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:10:52.564028573Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:10:53.226072143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-04T13:10:54.44128427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:10:55.314611748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:10:55.518796357Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-04T13:10:55.51881981Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-04T13:10:56.186213427Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:10:58.642206843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:10:58.645521166Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T13:10:58.936845047Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-04T13:10:59.441177365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:11:02.354266617Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-04T13:11:03.478534561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:11:03.482402192Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:11:04.441439407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:05.163970184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:11:05.168018861Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-04T13:11:05.771951101Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-04T13:11:09.189310732Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-04T13:11:09.442164679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:12.247811476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:11:12.255660902Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:11:12.60711402Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-04T13:11:14.441745977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:11:16.025099285Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-04T13:11:16.025130981Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-04T13:11:19.441400287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:11:19.441431356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:11:19.4414602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:11:19.441496581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:19.448699327Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-04T13:11:19.450083117Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008017617s
level=info ts=2026-05-04T13:11:19.698589004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:11:19.702693063Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-04T13:11:20.282244761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:11:20.286363376Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:11:23.084427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:11:23.090889458Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-04T13:11:24.442050316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:24.803305289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:11:24.807460445Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-04T13:11:25.315731924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T13:11:25.950218517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:11:25.954479404Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T13:11:27.587121377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:11:27.592902302Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:11:27.592925822Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:11:27.59295339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:11:27.598132195Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:11:28.330517692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:11:28.503057128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:11:28.511249428Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-04T13:11:29.441785076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:32.503079229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:11:32.507920781Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:11:34.26031451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:11:34.266849956Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-04T13:11:34.441338575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:36.629944187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:11:36.635432641Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:11:37.211598814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:11:37.217332082Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:11:38.507960825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:11:38.511896459Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=warn ts=2026-05-04T13:11:39.441410481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:40.781242386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:11:40.786487711Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:11:41.275007578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:11:41.278575508Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:11:42.179408129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:11:42.182976111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-04T13:11:44.442112164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:11:44.466314797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:11:44.470501461Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:11:44.470526461Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:11:44.470550171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:11:44.473652452Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T13:11:45.03908285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:11:45.042667396Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-04T13:11:46.675729159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:40084"
ts=2026-05-04T13:11:47.808583582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:39868"
level=debug ts=2026-05-04T13:11:49.423712348Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:11:49.424753947Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:11:49.441036244Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-04T13:11:49.441236826Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-04T13:11:49.441246832Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-04T13:11:49.441267014Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.872µs success=true
level=info ts=2026-05-04T13:11:49.441271987Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:11:49.441299627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:11:49.441307564Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:11:49.441323316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:11:49.441516849Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-04T13:11:51.346766668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:11:51.356105072Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:11:52.859792281Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-04T13:11:54.441675795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:11:55.317489766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=debug ts=2026-05-04T13:11:56.277372673Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-04T13:11:57.164801069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:11:58.113587355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:11:58.117543355Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-04T13:11:59.441585358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:11:59.695124412Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-04T13:12:02.498432059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:12:02.504658579Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:12:03.113387858Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-04T13:12:03.976833688Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:12:03.976846204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-04T13:12:03.980924119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df2d4eaaa%3A19df2e66bcd%3A13ef540e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-04T13:12:04.019166059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:12:04.024572357Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:12:04.278220619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:12:04.282074822Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-04T13:12:04.441490044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:12:06.512050628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:12:06.515731295Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-04T13:12:06.530807475Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-04T13:12:09.44188849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:12:09.948306387Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T13:12:12.186373378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:12:12.190759417Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:12:13.366085179Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-04T13:12:14.434741987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:12:14.440687701Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:12:14.441661253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:12:16.67720934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51498"
level=debug ts=2026-05-04T13:12:16.784082591Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-04T13:12:17.810461119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:43014"
level=info ts=2026-05-04T13:12:18.552992164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-04T13:12:18.557486399Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df2d4eaaa%3A19df2e66bcd%3A13ef540e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-04T13:12:18.56153253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:12:18.565593327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:12:19.441241255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:12:19.441288753Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:12:19.441312913Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:12:19.441322546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T13:12:19.764182034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:12:19.769383875Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T13:12:20.201340508Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-04T13:12:20.739052061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:12:20.743115477Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:12:20.743141588Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:12:20.743166369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:12:20.746303938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T13:12:22.850713426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:12:22.85582169Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-04T13:12:23.61915075Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-04T13:12:24.441658187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:12:24.896217527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:12:24.900323748Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:12:24.900346459Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:12:24.900369502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:12:24.903415849Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
ts=2026-05-04T13:12:25.319058552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T13:12:25.609522335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:12:25.616341796Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:12:27.036824057Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-04T13:12:27.036847397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-04T13:12:27.769682033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:12:27.776405139Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:12:28.319008507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:12:28.325253374Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:12:29.441586985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:12:29.495973873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:12:30.456647926Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-04T13:12:30.45666939Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=info ts=2026-05-04T13:12:31.0118288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:12:31.015721657Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:12:31.015745552Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:12:31.015771335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:12:31.0188829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:12:31.138233531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="544 B" avg_comp="544 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:12:31.142076573Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:12:31.142104436Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:12:31.316626197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:12:31.320414409Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:12:31.340486248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:12:31.344958671Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-04T13:12:31.932799595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:12:31.936495342Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T13:12:33.873008927Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-04T13:12:34.441937239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:12:36.537450307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:12:36.5428224Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=info ts=2026-05-04T13:12:37.250908125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:12:37.258857872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:12:37.28995613Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-04T13:12:37.693242731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.1 kB" avg_comp="2.1 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:12:37.697026738Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:12:39.441952523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:12:40.708261814Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-04T13:12:42.529776583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:12:42.533174456Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-04T13:12:44.126459378Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-04T13:12:44.126481212Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=warn ts=2026-05-04T13:12:44.442129415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:12:46.678472999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:53286"
level=debug ts=2026-05-04T13:12:47.543977193Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=info ts=2026-05-04T13:12:48.498927739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:12:48.502805955Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T13:12:49.42301444Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:12:49.424302338Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:12:49.441676893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:12:49.441699883Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:12:49.441709064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:12:49.441735503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:12:49.837045809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-04T13:12:50.96074346Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-04T13:12:52.517213621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:12:52.520794507Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-04T13:12:53.755656665Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-04T13:12:54.288302334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59640"
level=debug ts=2026-05-04T13:12:54.379112468Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-04T13:12:54.441401771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:12:54.60090158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:12:54.606971624Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-04T13:12:55.321834243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:12:56.375243048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-04T13:12:56.378724942Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df2d4eaaa%3A19df2e66bcd%3A13ef540e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-04T13:12:57.796033832Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-04T13:12:59.441376008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:12:59.727161479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:12:59.733304406Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:13:01.217382999Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-04T13:13:01.217404398Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=5
level=info ts=2026-05-04T13:13:03.445692661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:13:03.451764915Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:13:03.558401586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-04T13:13:03.562949214Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df2d5505d%3A19df2d5a120%3A51ef14ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-04T13:13:03.562981052Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:13:03.563008734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-04T13:13:03.567261274Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df2d491a5%3A19df31b8521%3Aa3a04bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-04T13:13:04.441854582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:04.632109499Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T13:13:05.620531618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:13:05.624678541Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:13:08.050098459Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-04T13:13:08.880856381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:13:08.884636335Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=warn ts=2026-05-04T13:13:09.441510306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:13:10.436091007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:13:10.440540884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:13:10.484605196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:13:10.491446732Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:13:11.479584894Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T13:13:11.479610621Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-05-04T13:13:12.171447144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:13:12.175499264Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:13:12.175523988Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:13:12.17555559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:13:12.178731504Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:13:12.280111835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:13:12.283679686Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:13:12.283702636Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:13:12.38090863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:13:12.387412702Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:13:12.475959293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:13:12.479689295Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:13:13.924253172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:13:13.928245225Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-04T13:13:14.441322175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:14.885560432Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-04T13:13:18.302913846Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=warn ts=2026-05-04T13:13:19.441600841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:13:19.441607566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:13:19.44163068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:13:19.441641203Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-04T13:13:20.755867793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-04T13:13:20.762308038Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df2d491a5%3A19df31b8521%3Aa3a04bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-04T13:13:21.720480303Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
level=info ts=2026-05-04T13:13:21.905626718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:13:21.9093463Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:13:22.706659114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:13:22.712852144Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:13:22.849388962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:13:22.854384252Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-04T13:13:24.075236837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:13:24.078985106Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-04T13:13:24.291398713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:45676"
level=info ts=2026-05-04T13:13:24.426273387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="399 B" avg_comp="399 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:13:24.430891565Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-04T13:13:24.442084994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:25.138266964Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
ts=2026-05-04T13:13:25.324616776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:13:26.037336098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:13:26.043125095Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:13:28.200447667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:13:28.207489533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:13:28.207515322Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:13:28.207545957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:13:28.214241288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:13:28.562486144Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-04T13:13:28.562510484Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=5
level=debug ts=2026-05-04T13:13:28.796822221Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:13:28.796881417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="415 kB" avg_comp="415 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-04T13:13:28.801177377Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df2d49740%3A19df315ebf5%3A3d273111?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-04T13:13:29.222830512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:13:29.226397884Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:13:29.442179156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:31.973657944Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-04T13:13:34.441534811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:35.390943047Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-04T13:13:36.664063895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-04T13:13:36.668223405Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df2d4eaaa%3A19df2e66bcd%3A13ef540e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-04T13:13:37.768614673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:13:37.772136955Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-04T13:13:38.809435526Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-04T13:13:39.442170068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:13:40.433893256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:13:40.438732019Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:13:40.621315038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:13:40.628900123Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:13:42.226387172Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-04T13:13:42.590932955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-04T13:13:44.441547324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:45.644272161Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-04T13:13:46.681408443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:51636"
level=info ts=2026-05-04T13:13:47.104243033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:13:47.109425943Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:13:47.174624333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-04T13:13:47.178665561Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df2d491a5%3A19df31b8521%3Aa3a04bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-04T13:13:47.818372807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.56:36900"
level=info ts=2026-05-04T13:13:48.611907394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:13:48.617171289Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:13:48.633362092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-04T13:13:48.637199913Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df2d49740%3A19df315ebf5%3A3d273111?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-04T13:13:49.062535421Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-04T13:13:49.423089092Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:13:49.424629439Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:13:49.442019982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T13:13:49.442032274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:13:49.442053708Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:13:49.442063734Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T13:13:52.479661352Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-04T13:13:53.580895135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:13:53.585893206Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:13:53.937570786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:13:53.941900075Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T13:13:54.4417854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:13:54.895727986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:13:54.899303311Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
ts=2026-05-04T13:13:55.327071344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-cfvrk-cd059455 10.129.0.56:7946"
level=debug ts=2026-05-04T13:13:55.897822436Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-04T13:13:57.118259659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:13:57.122783437Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-04T13:13:57.441948647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:13:57.447939976Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:13:57.447962678Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:13:57.447986608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:13:57.453077336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:13:59.172308443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:13:59.176173221Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T13:13:59.315763277Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-04T13:13:59.441989677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:13:59.590242261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:13:59.596275281Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:14:01.966745203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:14:01.970265573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-04T13:14:02.72768694Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:14:02.732973695Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=debug ts=2026-05-04T13:14:02.732994684Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=10
level=info ts=2026-05-04T13:14:03.409448677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:14:03.415165414Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:14:04.441482996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:06.150633154Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-04T13:14:07.164674913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:14:07.168700356Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-04T13:14:07.508782966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:14:07.513694065Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-04T13:14:09.441794718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:09.568464814Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-04T13:14:09.573374735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-04T13:14:09.577872103Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df2d49740%3A19df315ebf5%3A3d273111?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-04T13:14:10.272972311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:14:10.280292763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-04T13:14:10.552225621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:14:10.556022834Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:14:10.556046538Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:14:10.556071832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:14:10.559337667Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-04T13:14:12.986978274Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=warn ts=2026-05-04T13:14:14.441728993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:14:14.68438859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:14:14.688449391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:14:15.950415844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:14:15.956745601Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-04T13:14:16.40610711Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-04T13:14:16.406129883Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=4
level=info ts=2026-05-04T13:14:19.441954599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T13:14:19.441969673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:19.441989086Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:14:19.441997706Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T13:14:19.461399977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-04T13:14:19.463099332Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-04T13:14:19.821813223Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-04T13:14:21.306065589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:14:21.310469378Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-04T13:14:23.24132985Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-04T13:14:23.241352762Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-05-04T13:14:23.867561171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:14:23.873059614Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-04T13:14:24.046903424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-04T13:14:24.050312158Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df2d4eaaa%3A19df2e66bcd%3A13ef540e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-04T13:14:24.297939744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:44804"
level=warn ts=2026-05-04T13:14:24.441748915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-04T13:14:25.329065312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=debug ts=2026-05-04T13:14:26.657635264Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=warn ts=2026-05-04T13:14:29.442082218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:30.075148785Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-04T13:14:31.51652872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:14:31.522912603Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=debug ts=2026-05-04T13:14:33.493354077Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-04T13:14:34.442058944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:34.697437768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:14:36.910499475Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=info ts=2026-05-04T13:14:37.596604708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:14:37.600879548Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-04T13:14:38.073314487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:14:38.07737187Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-04T13:14:38.728707516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:14:38.732440389Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-04T13:14:39.441401143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:40.328368369Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-04T13:14:41.450765036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-04T13:14:41.455282977Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19df2d491a5%3A19df31b8521%3Aa3a04bb3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-04T13:14:43.098840315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:14:43.102275258Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-04T13:14:43.746281401Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-04T13:14:44.442097473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:14:45.506307377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:14:45.51237869Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:14:45.512405201Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:14:45.512432041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:14:45.517288042Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-04T13:14:45.683716065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ac9c403a73642fdd immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-shhc\"}"
level=error ts=2026-05-04T13:14:45.687171058Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac9c403a73642fdd/19df2df0378%3A19df2e0a73e%3A9b2b7e75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-shhc\"}"
level=debug ts=2026-05-04T13:14:47.164064539Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-04T13:14:49.423029098Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.57:11211 count=1
level=info ts=2026-05-04T13:14:49.424419799Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-04T13:14:49.441778748Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-04T13:14:49.441806396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:14:49.441814241Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-04T13:14:49.441812217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:14:50.581709547Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-04T13:14:51.867565722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:14:51.873957105Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:14:51.873986895Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:14:51.874014888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:14:51.879658397Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-04T13:14:54.001970356Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-04T13:14:54.001994322Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=9
ts=2026-05-04T13:14:54.30124412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:59272"
level=warn ts=2026-05-04T13:14:54.442078715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:14:55.162122275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="416 kB" avg_comp="416 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-04T13:14:55.168040989Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df2d49740%3A19df315ebf5%3A3d273111?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-04T13:14:55.331824971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-2dr4k-dd36ae67 10.128.0.56:7946"
level=info ts=2026-05-04T13:14:55.722899391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-04T13:14:55.729122844Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df2d4eaaa%3A19df2f52e1d%3Ad4294bf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-04T13:14:56.312820141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:14:56.318327027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:14:56.318355011Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:14:56.318382147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:14:56.32311953Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-04T13:14:56.737471114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="134 kB" avg_uncomp="134 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-04T13:14:56.740911889Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df2d48cbf%3A19df2d7ec71%3A281e4657?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-04T13:14:57.416992318Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=info ts=2026-05-04T13:14:57.685710111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="562 B" avg_comp="562 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-04T13:14:57.689330232Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df2d4eaaa%3A19df2d4ebfa%3A167401ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-04T13:14:58.11243388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:14:58.117002532Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:14:58.117027026Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-04T13:14:58.117054475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:14:58.12088351Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-04T13:14:58.301243156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.2 kB" avg_comp="2.2 kB" total_uncomp="6.2 kB" avg_uncomp="6.2 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-04T13:14:58.304697055Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19df2d4ebe9%3A19df2dc68e0%3Ad3c0b073?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-04T13:14:59.441953592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:15:00.43423633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="417 B" avg_comp="417 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-04T13:15:00.438544878Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df2d5fee0%3A19df2d5fee1%3A8179ee56?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-04T13:15:00.796450003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:15:00.802611986Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-04T13:15:00.834636456Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-04T13:15:00.834660076Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=debug ts=2026-05-04T13:15:01.516123258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-04T13:15:04.252662809Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-04T13:15:04.44203404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:15:04.816115701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-04T13:15:04.820215914Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df2d49187%3A19df2d4eaab%3Ad618a924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-04T13:15:05.285282416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-04T13:15:05.469990057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="24 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-04T13:15:05.476005854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df2d492bf%3A19df2e98c96%3Ae8392323?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-04T13:15:06.919403356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-04T13:15:06.927039222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df2d48d51%3A19df2d601b4%3A58e15ec8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-04T13:15:07.670319351Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-04T13:15:08.746237408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="9.0 MB" avg_uncomp="9.0 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-04T13:15:08.752115602Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19df2d4eaaa%3A19df2f18046%3A18dd2374?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-04T13:15:09.441779658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:15:11.088396159Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
level=info ts=2026-05-04T13:15:13.401867416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-04T13:15:13.405652277Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df2d4eaaa%3A19df2e66bcd%3A13ef540e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-04T13:15:14.441964136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:15:14.505334278Z caller=checkpoint.go:372 msg="writing series" size="492 kB"
level=debug ts=2026-05-04T13:15:14.505357189Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-04T13:15:15.423999767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="426 kB" avg_comp="426 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-04T13:15:15.428637408Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df2d4eaaa%3A19df2e72ac9%3Ad5d74a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-04T13:15:16.554728272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="672 kB" avg_comp="672 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-04T13:15:16.559501504Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df2d48bb2%3A19df314d2b6%3A6a0c0c82?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-04T13:15:16.689042709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.59:36130"
level=debug ts=2026-05-04T13:15:17.923945151Z caller=checkpoint.go:372 msg="writing series" size="754 kB"
level=info ts=2026-05-04T13:15:19.441715731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-04T13:15:19.44173157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:15:19.441746962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-04T13:15:19.441755686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-04T13:15:21.341117529Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-04T13:15:21.962840876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-04T13:15:21.96866261Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df2d48bce%3A19df301a716%3A5b645370?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-04T13:15:24.304668809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.56:46120"
level=info ts=2026-05-04T13:15:24.311227961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-04T13:15:24.314684189Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df2d4ebe8%3A19df2d4ebfa%3A54103962?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-04T13:15:24.441887597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-04T13:15:24.758424298Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=debug ts=2026-05-04T13:15:24.758451556Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
ts=2026-05-04T13:15:25.334376139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6fcd9c02 10.129.0.59:7946"
level=info ts=2026-05-04T13:15:26.426552153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-04T13:15:26.43248337Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df2d4eaaa%3A19df2d54b30%3A6084a798?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-04T13:15:28.176318528Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=warn ts=2026-05-04T13:15:29.4420874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:15:29.681312457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=714e8f183a8bd3be immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-furw\"}"
level=error ts=2026-05-04T13:15:29.687736809Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/714e8f183a8bd3be/19df2dec2dd%3A19df30ae5d8%3A26b2f165?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-furw\"}"
level=info ts=2026-05-04T13:15:29.848405171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-04T13:15:29.852261617Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df2d4eaaa%3A19df2d52913%3A9a59f57e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-04T13:15:30.497011824Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-04T13:15:30.497075575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=df322c95c414aaa8 immediate=false num_chunks=1 total_comp="351 kB" avg_comp="351 kB" total_uncomp="1.5 MB" avg_uncomp="1.5 MB" idle=1 labels="{stream=\"build-e2e-idgs\"}"
level=error ts=2026-05-04T13:15:30.507877051Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/df322c95c414aaa8/19df2de9bcc%3A19df2ea0d11%3Af0dca441?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-idgs\"}"
level=debug ts=2026-05-04T13:15:31.594076096Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-04T13:15:34.442135167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-04T13:15:34.5404957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-04T13:15:34.544335021Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df2de9617%3A19df2dfe220%3A40135564?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-04T13:15:35.01247388Z caller=checkpoint.go:372 msg="writing series" size="574 kB"