level=info ts=2026-04-21T04:01:53.1639744Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T04:01:53.164000343Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-21T04:01:53.164321818Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-21T04:01:53.165296561Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T04:01:53.165661527Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T04:01:53.16572747Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.79 inf=eth0
level=info ts=2026-04-21T04:01:53.166309421Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-80814b76
ts=2026-04-21T04:01:53.166375832Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T04:01:53.166390164Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.79 advertisePort=7946
level=debug ts=2026-04-21T04:01:53.166416072Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.79 advertisePort=7946
level=info ts=2026-04-21T04:01:53.166533244Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-21T04:01:53.17142739Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:01:53.171461562Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-04-21T04:01:53.172332157Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T04:01:53.172357514Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:01:53.172373406Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=5.841364ms
level=error ts=2026-04-21T04:01:53.172395315Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-21T04:01:53.172405945Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-21T04:01:53.172439981Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-21T04:01:53.172960219Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:01:53.173401728Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-21T04:01:53.173447383Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-21T04:01:53.173443503Z
level=debug ts=2026-04-21T04:01:53.173473284Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-21T04:01:53.173548345Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-21T04:01:53.173572472Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-21T04:01:53.173579821Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-21T04:01:53.173578708Z
level=debug ts=2026-04-21T04:01:53.173585038Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-21T04:01:53.174596647Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T04:01:53.174670702Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.79 inf=eth0
ts=2026-04-21T04:01:53.175013272Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:01:53.175025512Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-21T04:01:53.188135942Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=info ts=2026-04-21T04:01:53.188125764Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T04:01:53.188145821Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-21T04:01:53.18815254Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-04-21T04:01:53.188160973Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-21T04:01:53.188162276Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=info ts=2026-04-21T04:01:53.188168077Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-04-21T04:01:53.188180977Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-21T04:01:53.188244417Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-21T04:01:53.188251285Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-21T04:01:53.188244407Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-21T04:01:53.188426388Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-04-21T04:01:53.188424824Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-21T04:01:53.188432536Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-21T04:01:53.188435349Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-04-21T04:01:53.188439257Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-21T04:01:53.18845109Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-21T04:01:53.188483192Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-21T04:01:53.188489705Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-21T04:01:53.188493969Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-21T04:01:53.188498822Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-21T04:01:53.188503454Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-21T04:01:53.188507537Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-04-21T04:01:53.188511308Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-21T04:01:53.188603247Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-21T04:01:53.188627886Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-21T04:01:53.188655488Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-21T04:01:53.188756348Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=130.033µs errors=false
level=info ts=2026-04-21T04:01:53.188766452Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-21T04:01:53.188933104Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=304.787µs errors=false
level=info ts=2026-04-21T04:01:53.188951371Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-21T04:01:53.188962838Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=336.291µs
level=info ts=2026-04-21T04:01:53.188980953Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-21T04:01:53.189016537Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T04:01:53.189045668Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-21T04:01:53.189071742Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=7.458662369s
level=debug ts=2026-04-21T04:01:53.189121012Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-21T04:01:53.189126822Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-04-21T04:01:53.189152671Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=debug ts=2026-04-21T04:01:53.189162182Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-21T04:01:53.189165087Z caller=loki.go:599 msg="Loki started" startup_time=86.17516ms
ts=2026-04-21T04:01:54.418620909Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:01:54.418653155Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:01:58.189375939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:01:58.215705029Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:01:58.215722576Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:02:02.243378512Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:02:02.243402878Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:02:03.18943024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:02:08.190064164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:02:09.296584897Z caller=logging.go:137 msg="GET /ready (503) 98.633µs"
level=warn ts=2026-04-21T04:02:13.189707644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:02:15.360596067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.80:7946"
level=info ts=2026-04-21T04:02:15.361296042Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=22.188883491s
level=warn ts=2026-04-21T04:02:18.189252899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:02:19.295366477Z caller=logging.go:137 msg="GET /ready (503) 75.707µs"
level=info ts=2026-04-21T04:02:23.190156106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T04:02:23.190184082Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-21T04:02:23.190193279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:02:23.190182373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:02:27.871857813Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:02:28.189948833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:02:33.1899478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:02:36.949547334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:02:38.190101053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:02:43.1900218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:02:44.031752138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51840"
level=debug ts=2026-04-21T04:02:44.813253454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:02:48.189575865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:02:52.742272651Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:02:52.742365367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:02:52.748366634Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:02:53.173013423Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:02:53.173366943Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:02:53.189710083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:02:53.189715518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:02:53.189729756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:02:53.189740875Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:02:54.85114823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50956"
level=warn ts=2026-04-21T04:02:58.190116659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:03:02.393783685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:03:03.190130526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:03:06.951590347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:03:08.189428234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:03:08.528657458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:03:08.537368307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:03:13.189548616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:03:14.034592907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39226"
level=warn ts=2026-04-21T04:03:18.189782483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:03:19.692008103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:03:23.189435536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:03:23.189454325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:03:23.189477105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:03:23.189488237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:03:26.300060447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:03:28.18934016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:03:33.189720979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:03:36.198835535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:03:36.206377843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:03:36.460182005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:03:36.954253347Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:03:38.189349296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:03:43.190198131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:03:48.189345988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:03:53.173016562Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:03:53.173526382Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:03:53.189799306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:03:53.189814567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:03:53.189822252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:03:53.189837622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:03:54.282530449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:03:54.854842104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37262"
level=warn ts=2026-04-21T04:03:58.189949046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:04:03.189383605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:04:06.706697452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T04:04:06.708347991Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:04:06.957082528Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:04:08.189218935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:04:13.190136292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:04:14.040088375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42034"
level=warn ts=2026-04-21T04:04:18.190064339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:04:23.189803023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:04:23.189796123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:04:23.189827448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:04:23.189838621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:04:27.01299822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:04:27.021636642Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:04:28.090431149Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:04:28.189559667Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:04:33.189766999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:04:36.959567732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:04:38.189776933Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:04:43.189256773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:04:44.042828842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56264"
level=warn ts=2026-04-21T04:04:48.189840293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:04:53.17303459Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:04:53.173409192Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:04:53.189675738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:04:53.18969538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:04:53.189703709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:04:53.189701765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:04:54.86114132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58580"
level=debug ts=2026-04-21T04:04:58.185412795Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:04:58.189474381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:04:59.482427274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:05:03.189976485Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:05:06.962451705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:05:08.189519105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:05:13.189733877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:05:18.190240994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:05:23.189335117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:05:23.189365805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:05:23.189367923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:05:23.189384277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:05:24.864476178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59138"
level=info ts=2026-04-21T04:05:25.806580835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:05:25.817323478Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:05:28.18964469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:05:33.18996843Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:05:33.877429472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:05:36.965907683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:05:38.189501919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:05:43.189867846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:05:48.189943455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:05:53.173176016Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:05:53.173181214Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:05:53.189437853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:05:53.189464948Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:05:53.189490669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:05:53.18950514Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:05:58.189650731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:06:03.190076996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:06:05.066642638Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:06:06.622486226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:06:06.62994706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:06:06.969082765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:06:08.189664429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:06:11.296184257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:06:13.189943086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:06:14.049001696Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35080"
level=warn ts=2026-04-21T04:06:18.189357958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:06:23.189159489Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:06:23.189192585Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:06:23.189202834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:06:23.18919187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:06:28.190160852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:06:33.189476757Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:06:33.501703873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:06:36.971958681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:06:38.190043825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:06:43.189348088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:06:44.051720357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51320"
level=warn ts=2026-04-21T04:06:48.189658448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:06:53.172714085Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:06:53.1737551Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:06:53.189029932Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T04:06:53.189154201Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-21T04:06:53.189225799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:06:53.189248755Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:06:53.189259872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:06:53.189250669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:06:53.189546488Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
level=info ts=2026-04-21T04:06:55.038643237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:06:55.045073864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:06:57.544430659Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:06:58.189845609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:01.899935586Z caller=checkpoint.go:378 msg="writing series" size="299 kB"
level=warn ts=2026-04-21T04:07:03.189242681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:06.254478692Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:07:06.973595371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:07:07.63007365Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:07:07.958619245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T04:07:08.189576523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:10.609740453Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-21T04:07:13.189261613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:07:14.054258732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59516"
level=debug ts=2026-04-21T04:07:14.964127316Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
ts=2026-04-21T04:07:15.55919971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59270"
level=warn ts=2026-04-21T04:07:18.189584444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:19.318938307Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-04-21T04:07:23.189808397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:07:23.189835028Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:07:23.189843363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:07:23.189874008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:23.673986798Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
ts=2026-04-21T04:07:24.875098111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39848"
level=debug ts=2026-04-21T04:07:28.02890414Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:07:28.189653634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:32.383808829Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:07:33.189288681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:34.082960444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:07:36.362432416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:07:36.37227063Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:07:36.738666886Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-21T04:07:36.974768184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:07:38.189263578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:41.093017807Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-04-21T04:07:41.093044833Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-21T04:07:43.189657289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:07:44.057202275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33806"
level=debug ts=2026-04-21T04:07:45.448024781Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-21T04:07:48.189759905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:49.803044782Z caller=checkpoint.go:378 msg="writing series" size="440 kB"
level=info ts=2026-04-21T04:07:53.173388982Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:07:53.17382636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:07:53.190127998Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:07:53.190145235Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:07:53.190152913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:07:53.190165859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:54.157744654Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
ts=2026-04-21T04:07:54.878310322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45094"
level=warn ts=2026-04-21T04:07:58.189904907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:07:58.513043284Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=debug ts=2026-04-21T04:08:02.867094898Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:08:03.189983834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:05.477383062Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:08:06.976393454Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:08:07.221863763Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:08:08.189674368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:11.577534648Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:08:13.189940096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:08:14.059795609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56542"
level=debug ts=2026-04-21T04:08:15.153806243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T04:08:15.563198878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:36038"
level=debug ts=2026-04-21T04:08:15.931421378Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:08:18.189330765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:20.286613976Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:08:23.18969928Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:08:23.1897303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:08:23.189740574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:08:23.189812435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:24.641720329Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=debug ts=2026-04-21T04:08:24.641744677Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-21T04:08:28.190177069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:28.997490969Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-21T04:08:28.997517629Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-04-21T04:08:29.7024949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:08:29.709903931Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:08:31.026524721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:08:33.189778435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:33.351178373Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
ts=2026-04-21T04:08:36.977424162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:08:37.705984329Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:08:38.189581464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:42.062350028Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:08:43.189367171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:46.415786111Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-21T04:08:48.189649447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:50.771056384Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=info ts=2026-04-21T04:08:53.173025522Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:08:53.173566216Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:08:53.189839137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:08:53.189843361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:53.189861677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:08:53.18986953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:08:54.883113327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45740"
level=debug ts=2026-04-21T04:08:55.124994728Z caller=checkpoint.go:378 msg="writing series" size="215 kB"
level=warn ts=2026-04-21T04:08:58.189505861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:08:59.480287321Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-21T04:09:03.189279422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:03.835471165Z caller=checkpoint.go:378 msg="writing series" size="1.6 kB"
level=debug ts=2026-04-21T04:09:05.473999626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:09:06.979442873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:09:08.190222748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:08.190313806Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=debug ts=2026-04-21T04:09:12.545221534Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-21T04:09:13.189777375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:16.899710212Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:09:18.190186866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:21.253964302Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:09:21.782623759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:09:21.789296179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:09:21.78931683Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:09:21.789351984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:09:21.795288476Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:09:23.189633886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:09:23.189660264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:09:23.189669306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:09:23.189684493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:09:24.88560074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37958"
level=debug ts=2026-04-21T04:09:25.609665272Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:09:28.189912708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:29.963816698Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:09:33.190243174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:34.318762031Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-21T04:09:34.350958844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T04:09:34.352637772Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:09:35.466682879Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:09:36.981403411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:09:38.189532805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:38.673291954Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T04:09:40.152927438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:09:40.162094974Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:09:43.028831604Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:09:43.189683911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:47.3837266Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:09:48.189995032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:51.738230355Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T04:09:53.173041929Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:09:53.173348603Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:09:53.18971859Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:09:53.189736526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:09:53.189745894Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:09:53.189832036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:09:56.093580995Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=debug ts=2026-04-21T04:09:56.093604894Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=20
level=warn ts=2026-04-21T04:09:58.189869129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:00.448384667Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:10:03.190010218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:04.80270843Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:10:06.982665268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:10:07.018374951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:10:08.19006285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:09.157527999Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-04-21T04:10:12.28975091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:10:12.297819528Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:10:13.189759744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:13.513022518Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-21T04:10:16.090186912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:10:17.867538179Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:10:18.190154208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:22.221960074Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:10:23.189909141Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:10:23.189933009Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:10:23.189940792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:10:23.189940578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:10:24.890064058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:33120"
level=debug ts=2026-04-21T04:10:26.577112789Z caller=checkpoint.go:378 msg="writing series" size="362 kB"
level=warn ts=2026-04-21T04:10:28.190028185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:30.935461091Z caller=checkpoint.go:378 msg="writing series" size="5.5 MB"
level=debug ts=2026-04-21T04:10:30.935483571Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.2 MB" series=8
level=warn ts=2026-04-21T04:10:33.189550277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:35.286530146Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
ts=2026-04-21T04:10:36.984152653Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:10:38.189643278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:39.641218689Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=debug ts=2026-04-21T04:10:41.694821011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:10:43.189233572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:43.996479562Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
ts=2026-04-21T04:10:44.067955298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34256"
ts=2026-04-21T04:10:45.575965866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:60910"
level=warn ts=2026-04-21T04:10:48.189575005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:48.350967911Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-21T04:10:52.705907353Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-21T04:10:53.172645955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:10:53.173686206Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:10:53.189949776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:10:53.189964795Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:10:53.189973265Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:10:53.189976603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:10:55.203804668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:10:57.060809667Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=info ts=2026-04-21T04:10:57.984643133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:10:57.9927345Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:10:58.19008691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:11:01.41542565Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:11:03.189428583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:11:05.770597044Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
ts=2026-04-21T04:11:06.986815236Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:11:08.189852278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:11:10.125361115Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=debug ts=2026-04-21T04:11:10.337120723Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:11:13.189781995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:11:14.480437598Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-21T04:11:15.57739035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:39684"
level=warn ts=2026-04-21T04:11:18.190054983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:11:18.835560404Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=debug ts=2026-04-21T04:11:23.189628753Z caller=checkpoint.go:390 msg="flushing series" totalSize="766 kB" series=11
level=info ts=2026-04-21T04:11:23.189634248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:11:23.18965905Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:11:23.18966956Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:11:23.189678399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:11:23.19523296Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-21T04:11:23.199593738Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010252681s
ts=2026-04-21T04:11:24.895892289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48154"
level=warn ts=2026-04-21T04:11:28.189854011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:11:33.189983309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:11:36.989622585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:11:38.190178277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:11:41.683744552Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:11:42.10297825Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:11:43.189690457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:11:44.07189473Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39754"
level=warn ts=2026-04-21T04:11:48.189731033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:11:53.173048267Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:11:53.174069697Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:11:53.189232773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:11:53.189254175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:11:53.189263738Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:11:53.189270973Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T04:11:53.189274169Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:11:53.189430477Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-04-21T04:11:53.1898288Z caller=checkpoint.go:378 msg="writing series" size="453 kB"
level=info ts=2026-04-21T04:11:54.617559568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:11:54.62506922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:11:57.545292803Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:11:58.189652777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:01.899999461Z caller=checkpoint.go:378 msg="writing series" size="316 kB"
level=warn ts=2026-04-21T04:12:03.189585643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:06.254733582Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:12:06.992463307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:12:08.189666681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:10.609359155Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-21T04:12:13.189690418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:12:14.07493293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41076"
level=debug ts=2026-04-21T04:12:14.964422426Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-21T04:12:15.581126235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:39512"
level=debug ts=2026-04-21T04:12:18.050318376Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:12:18.189463393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:19.31899013Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-04-21T04:12:23.18919442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:12:23.189232009Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:12:23.189251528Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:12:23.189260075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:12:23.674306242Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=debug ts=2026-04-21T04:12:28.029243273Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:12:28.189955186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:32.383650832Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:12:33.189203406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:36.73840815Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=debug ts=2026-04-21T04:12:36.738433669Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
ts=2026-04-21T04:12:36.99450681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:12:38.189968287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:39.787564731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:12:41.09339602Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-21T04:12:43.18974192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:45.448415143Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
ts=2026-04-21T04:12:45.58244596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:41854"
level=info ts=2026-04-21T04:12:48.170235409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:12:48.176423528Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:12:48.189472305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:49.802893428Z caller=checkpoint.go:378 msg="writing series" size="433 kB"
level=debug ts=2026-04-21T04:12:50.858842063Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:12:53.172996389Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:12:53.173006772Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:12:53.189241432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:12:53.189261098Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:12:53.189271209Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:12:53.189320739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:54.157789675Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-21T04:12:58.190027121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:12:58.51324938Z caller=checkpoint.go:378 msg="writing series" size="322 kB"
level=debug ts=2026-04-21T04:13:02.867701259Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:13:03.189742823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:13:06.996620964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:13:07.221991437Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:13:08.18986645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:11.576900833Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:13:13.189327689Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:15.932373339Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=debug ts=2026-04-21T04:13:15.932401677Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-21T04:13:17.460854353Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:13:18.18934647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:20.287228936Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:13:23.190159124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:13:23.190188404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:13:23.190196744Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:13:23.190197764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:24.641893362Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=debug ts=2026-04-21T04:13:27.445106724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:13:27.446884412Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:13:28.189193605Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:28.998280331Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-21T04:13:28.998308337Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-04-21T04:13:33.190013657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:33.35135805Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=info ts=2026-04-21T04:13:35.179585922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:13:35.186476823Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:13:36.999674101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:13:37.706340329Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:13:38.189940952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:42.060764363Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:13:43.189946043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:46.062403726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:13:46.4160821Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-21T04:13:48.189517529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:50.770498529Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T04:13:53.173465443Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:13:53.174502843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:13:53.189713741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:13:53.1897318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:13:53.189741126Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:13:53.189742793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:55.126059098Z caller=checkpoint.go:378 msg="writing series" size="345 kB"
level=warn ts=2026-04-21T04:13:58.189564252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:13:59.480377576Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-21T04:14:03.189348346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:03.835748562Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-21T04:14:05.088645051Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:14:07.002343813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:14:08.1893822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:08.190660344Z caller=checkpoint.go:378 msg="writing series" size="303 kB"
level=debug ts=2026-04-21T04:14:08.190675445Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-21T04:14:12.545322374Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-21T04:14:13.189640879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:16.899832319Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:14:18.189685005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:19.191953438Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:14:19.917655748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:14:19.928370413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:14:21.254524852Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:14:23.189175459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:14:23.189202425Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:14:23.189228808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:14:23.189240917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:25.609091143Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:14:28.189473704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:29.964289593Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:14:33.189727925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:34.319504809Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:14:37.005114858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:14:38.190060459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:38.67397955Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:14:43.029301308Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:14:43.189255918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:14:45.5928345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:43240"
level=debug ts=2026-04-21T04:14:47.383850705Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:14:48.189934817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:49.946697593Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:14:51.207519507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:14:51.738886104Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-21T04:14:53.17312298Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:14:53.173130508Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:14:53.189464569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:14:53.189467127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:14:53.189491447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:14:53.189499629Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:14:54.909692825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36480"
level=debug ts=2026-04-21T04:14:56.093809946Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-04-21T04:14:58.190239536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:00.448766656Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:15:03.189946881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:04.803650741Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:15:07.007672772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:15:08.189645307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:09.158558222Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=info ts=2026-04-21T04:15:12.892346606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:15:12.900871544Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:15:13.189797769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:13.513044754Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T04:15:14.08938388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54344"
level=debug ts=2026-04-21T04:15:17.867845626Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:15:18.189554238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:20.14398857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:15:20.144073632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:15:20.151271007Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:15:22.222339703Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:15:23.189667508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:15:23.189695318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:15:23.189703114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:15:23.189739137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:15:24.912965222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42832"
level=debug ts=2026-04-21T04:15:26.577406588Z caller=checkpoint.go:378 msg="writing series" size="442 kB"
level=debug ts=2026-04-21T04:15:26.577429332Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=18
level=warn ts=2026-04-21T04:15:28.189589656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:29.871300995Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:15:30.936175886Z caller=checkpoint.go:378 msg="writing series" size="6.3 MB"
level=debug ts=2026-04-21T04:15:30.936200306Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.3 MB" series=1
level=warn ts=2026-04-21T04:15:33.189480625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:15:34.481150816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:15:34.487180377Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:15:35.286506114Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
ts=2026-04-21T04:15:37.011202905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:15:38.189783644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:39.641462286Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-21T04:15:43.189322102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:43.997142279Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
ts=2026-04-21T04:15:44.092848128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35478"
level=warn ts=2026-04-21T04:15:48.189996988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:48.351424361Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-21T04:15:52.705848145Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:15:53.17339706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:15:53.174132093Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=debug ts=2026-04-21T04:15:53.174333949Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T04:15:53.174375024Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974160 n=1
level=debug ts=2026-04-21T04:15:53.174416731Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974160 n=1
level=info ts=2026-04-21T04:15:53.189332478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:15:53.189317574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:15:53.189348369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:15:53.189356015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:15:54.328836268Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:15:56.702638731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:15:56.709308905Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:15:56.709343525Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:15:56.70936757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:15:56.715503385Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:15:57.061514019Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:15:58.189333391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:01.416038052Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:16:03.189493396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:05.771323836Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=info ts=2026-04-21T04:16:06.406321134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:16:06.413600381Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T04:16:07.013363132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:16:08.189447268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:10.125462584Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=warn ts=2026-04-21T04:16:13.189859068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:16:13.88641162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:16:13.893163034Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:16:14.095771753Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51594"
level=debug ts=2026-04-21T04:16:14.480364822Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-21T04:16:15.599340394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:48964"
level=debug ts=2026-04-21T04:16:17.008033152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:16:18.190258564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:18.835800217Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-21T04:16:18.904110625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:16:23.189161683Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:16:23.189189363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:16:23.189198355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:16:23.189226902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:23.190230996Z caller=checkpoint.go:390 msg="flushing series" totalSize="846 kB" series=11
level=info ts=2026-04-21T04:16:23.198833808Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-21T04:16:23.200245848Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010639138s
level=warn ts=2026-04-21T04:16:28.189824525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:16:33.189942759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:16:34.388417895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:16:34.397857961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:16:37.015309188Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:16:38.18964971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:16:43.189326191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:16:48.189515081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:48.836505911Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:16:52.314034323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:16:52.320855399Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:16:53.173018793Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:16:53.173014315Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:16:53.189321794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:16:53.189320249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:16:53.189340594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:16:53.189336569Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T04:16:53.189348817Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:16:53.189468428Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-21T04:16:53.189734912Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=debug ts=2026-04-21T04:16:57.545406365Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:16:58.189975342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:01.899553348Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=warn ts=2026-04-21T04:17:03.189406054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:06.254842383Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:17:07.018599229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:17:08.190104457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:09.542418413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:17:09.544133354Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:17:10.609616985Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-21T04:17:13.190166941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:14.96427295Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-21T04:17:15.377688345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:17:18.1899548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:19.318738926Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=info ts=2026-04-21T04:17:22.152330387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:17:22.160484065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:17:23.189836522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:17:23.189864075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:17:23.189872743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:17:23.18986429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:23.67432424Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=debug ts=2026-04-21T04:17:28.029246793Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:17:28.189940672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:32.383706083Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:17:33.189849352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:17:33.238143808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:17:33.245115517Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:17:36.738062498Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-21T04:17:37.021125665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:17:38.189954101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:41.093954211Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=debug ts=2026-04-21T04:17:41.09398249Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-04-21T04:17:42.165733581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:17:43.190154906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:45.448262685Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-04-21T04:17:48.190112794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:49.803511164Z caller=checkpoint.go:378 msg="writing series" size="433 kB"
level=info ts=2026-04-21T04:17:53.173097095Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:17:53.174012135Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:17:53.189259575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:17:53.189265299Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:17:53.189296009Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:17:53.189306303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:17:54.158049136Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-21T04:17:54.925477443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:60900"
level=debug ts=2026-04-21T04:17:57.581089481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:17:58.189851095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:17:58.512978278Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=debug ts=2026-04-21T04:18:02.86750136Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:18:03.189346559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:18:07.024149912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:18:07.222267483Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:18:08.189813786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:18:09.276067641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:18:09.283783322Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:18:11.577655262Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:18:13.189703005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:18:14.106291621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41330"
level=info ts=2026-04-21T04:18:14.940611036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:18:14.945661313Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:18:15.41231626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:18:15.932386762Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:18:18.1898658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:20.286943265Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:18:23.189537198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:18:23.189565438Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:18:23.189573678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:18:23.18957407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:24.642347325Z caller=checkpoint.go:378 msg="writing series" size="204 kB"
level=debug ts=2026-04-21T04:18:24.642373485Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-04-21T04:18:28.189937669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:28.997422953Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-21T04:18:28.997446556Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-04-21T04:18:33.189922687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:33.351379744Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
ts=2026-04-21T04:18:37.026724019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:18:37.706546602Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:18:38.189659502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:42.061151509Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:18:43.190014339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:45.897898333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:18:46.415677646Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=debug ts=2026-04-21T04:18:46.563767058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:18:48.189637084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:50.771117097Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-04-21T04:18:53.173005509Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:18:53.173241477Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:18:53.189540016Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:18:53.189546293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:18:53.189563717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:18:53.189572118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:18:54.93197128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54040"
level=debug ts=2026-04-21T04:18:55.125977979Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T04:18:58.190019472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:18:59.480950744Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-21T04:19:03.189656875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:03.835188001Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-04-21T04:19:05.556849292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:19:05.565897981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:19:07.027977712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:19:08.189498893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:08.190793999Z caller=checkpoint.go:378 msg="writing series" size="362 kB"
level=debug ts=2026-04-21T04:19:08.19080994Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-04-21T04:19:12.545454406Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=debug ts=2026-04-21T04:19:12.695015687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:19:12.978974182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:19:12.984243947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:19:13.189811545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:16.90008993Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:19:18.189790218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:21.254610529Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:19:23.189507248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:19:23.189532379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:19:23.189550522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:19:23.189580376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:25.60971109Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:19:28.189347277Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:29.964691525Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:19:33.190022758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:34.319275083Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:19:37.028704442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:19:38.189254984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:38.673837782Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:19:40.651739411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:19:43.028710407Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:19:43.189655257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:46.809384421Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:19:47.384128288Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:19:48.190168861Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:51.73898716Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=info ts=2026-04-21T04:19:53.173653501Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:19:53.174611785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:19:53.18983758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:19:53.189856467Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:19:53.189864702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:19:53.18987944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:19:56.093809209Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-21T04:19:58.189708807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:19:58.821264944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:19:58.82916628Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:20:00.448455461Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:20:03.189626202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:04.80280319Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:20:07.030002235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T04:20:07.947372332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:20:07.953461516Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:20:08.189664785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:09.157814847Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=debug ts=2026-04-21T04:20:11.480433899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:20:13.190177891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:13.513282214Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T04:20:15.621776934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:52598"
level=debug ts=2026-04-21T04:20:17.867714002Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:20:18.189577525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:22.222521739Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:20:23.189687375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:20:23.189721107Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:20:23.189705388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:20:23.189730038Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:20:26.577612335Z caller=checkpoint.go:378 msg="writing series" size="330 kB"
level=debug ts=2026-04-21T04:20:26.577637681Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=18
level=debug ts=2026-04-21T04:20:26.598332254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:20:28.189536467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:30.938012493Z caller=checkpoint.go:378 msg="writing series" size="6.8 MB"
level=debug ts=2026-04-21T04:20:30.938037545Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.8 MB" series=1
level=warn ts=2026-04-21T04:20:33.190108781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:35.286767561Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
ts=2026-04-21T04:20:37.030999616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:20:38.190193775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:39.641329655Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=debug ts=2026-04-21T04:20:40.987194948Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:20:40.987285184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="322 kB" avg_comp="322 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:20:40.992935828Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:20:43.189806009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:20:43.73899869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:20:43.746609376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:20:43.996508713Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
ts=2026-04-21T04:20:45.623601791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:49198"
level=warn ts=2026-04-21T04:20:48.189671362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:20:48.351049772Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-21T04:20:52.706279457Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:20:53.173066553Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:20:53.173064392Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:20:53.18938586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:20:53.189419969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:20:53.189421574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:20:53.189431703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:20:53.711192187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:20:53.715968695Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:20:57.061471473Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:20:58.190080133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:21:01.407256606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:21:01.413144173Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:21:01.416183505Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:21:03.189766825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:21:05.770870875Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
ts=2026-04-21T04:21:07.033758165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:21:08.189718794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:21:10.126122256Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=debug ts=2026-04-21T04:21:11.855204258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:21:13.18955891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:21:14.480388267Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-21T04:21:15.624618539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:35332"
level=info ts=2026-04-21T04:21:16.990645636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:21:16.996261584Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:21:18.189680329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:21:18.835143088Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=debug ts=2026-04-21T04:21:21.632374619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:21:21.634242043Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:21:23.189730233Z caller=checkpoint.go:390 msg="flushing series" totalSize="948 kB" series=11
level=info ts=2026-04-21T04:21:23.189749243Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:21:23.189765642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:21:23.189772996Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:21:23.189781217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:21:23.197463541Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-21T04:21:23.198964286Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009376922s
level=warn ts=2026-04-21T04:21:28.189294411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:21:33.190093648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:21:37.035808326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:21:38.189687077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:21:38.863063079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:21:42.817509147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:21:42.827867609Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:21:43.190160773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:21:45.626246913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:39850"
level=warn ts=2026-04-21T04:21:48.189593007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:21:53.173227482Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:21:53.173704551Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:21:53.189967205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T04:21:53.189977562Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-21T04:21:53.19000026Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:21:53.19000893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:21:53.190061147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:21:53.190127347Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-04-21T04:21:53.190462185Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
ts=2026-04-21T04:21:54.945418207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52290"
level=info ts=2026-04-21T04:21:57.43746446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:21:57.444147136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:21:57.444172566Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:21:57.444198705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:21:57.44918936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:21:57.546055873Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:21:58.189239007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:01.901191049Z caller=checkpoint.go:378 msg="writing series" size="369 kB"
level=warn ts=2026-04-21T04:22:03.189741522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:22:05.543667628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:22:05.548090904Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:22:06.255883074Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:22:07.037482187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:22:08.18990046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:22:08.562522361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:22:08.568056285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:22:09.890918583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:22:10.610236178Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-21T04:22:13.190078772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:14.965499516Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:22:18.189706047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:19.319988244Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-04-21T04:22:21.464748629Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T04:22:23.189943821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:22:23.189974088Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:22:23.189959521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:22:23.189982267Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:22:23.674932309Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
ts=2026-04-21T04:22:24.94837189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59918"
level=debug ts=2026-04-21T04:22:28.02974185Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:22:28.189605644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:32.38447401Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:22:33.189609405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:36.739267247Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-21T04:22:37.039488013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:22:37.755315768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:22:38.189541729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:22:40.66033921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:22:40.668040565Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:22:40.668063656Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:22:40.668101461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:22:40.674236215Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:22:41.094483427Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=debug ts=2026-04-21T04:22:41.094511686Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-04-21T04:22:41.487942648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:22:41.493609815Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:22:43.189616868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:45.449106807Z caller=checkpoint.go:378 msg="writing series" size="171 kB"
ts=2026-04-21T04:22:45.631057075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:48786"
level=warn ts=2026-04-21T04:22:48.189936495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:49.803758497Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=info ts=2026-04-21T04:22:53.097609784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:22:53.108120723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:22:53.173088223Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:22:53.173842875Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:22:53.189253854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:22:53.189252117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:22:53.189277887Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:22:53.189286356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:22:53.748641179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:22:53.752708651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:22:54.158255367Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-21T04:22:58.18922115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:22:58.513507052Z caller=checkpoint.go:378 msg="writing series" size="347 kB"
level=debug ts=2026-04-21T04:23:02.86849595Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:23:03.189337766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:06.145761544Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:23:07.041028484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:23:07.102114831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:23:07.223423829Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:23:08.189649516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:11.577562657Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:23:13.18946623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:15.932464042Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:23:18.189378263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:20.287658669Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:23:23.149131557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:23:23.161029042Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:23:23.18949425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:23:23.189533007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:23:23.189553247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:23:23.189570478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:24.642129343Z caller=checkpoint.go:378 msg="writing series" size="221 kB"
level=debug ts=2026-04-21T04:23:24.642155243Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-04-21T04:23:28.190005296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:28.997966449Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-21T04:23:28.997991847Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=1
level=warn ts=2026-04-21T04:23:33.189590108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:33.352048378Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=debug ts=2026-04-21T04:23:33.416614777Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:23:37.043459046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:23:37.70727672Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:23:38.190281268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:23:39.485950947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:23:39.491044921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:23:39.542140153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:23:39.54731877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:23:42.06210483Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:23:43.189445785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:46.417109163Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=warn ts=2026-04-21T04:23:48.189354019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:50.771947457Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-04-21T04:23:53.173008622Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:23:53.173310669Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:23:53.189645937Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:23:53.189669378Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:23:53.189677748Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:23:53.189694434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:55.126149251Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
level=warn ts=2026-04-21T04:23:58.189707204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:23:58.359100488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:23:59.481534073Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:24:03.189419338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:03.836102224Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
ts=2026-04-21T04:24:07.047041054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:24:07.607116545Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:24:08.189457081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:08.190633268Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-04-21T04:24:08.190650925Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-04-21T04:24:12.545754645Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=warn ts=2026-04-21T04:24:13.18984114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:24:14.133778313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55782"
ts=2026-04-21T04:24:15.638191374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:47758"
level=debug ts=2026-04-21T04:24:16.900739279Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T04:24:17.204814657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:24:17.211923008Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:24:18.190297702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:21.255898564Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:24:23.189873275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:24:23.189870677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:23.189894734Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:24:23.189902946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:24:25.610379967Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T04:24:27.841090788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:24:27.847749058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:24:27.849383263Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:24:27.850981937Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:24:28.189649364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:28.225874381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:24:29.964950868Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:24:33.189245854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:34.320357914Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:24:37.050594246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:24:38.189613099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:38.675140775Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:24:43.029374075Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:24:43.189800742Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:24:44.137134974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49568"
level=debug ts=2026-04-21T04:24:47.384471986Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:24:48.189967115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:51.739830213Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-21T04:24:53.173796069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:24:53.174307523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:24:53.189657907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:24:53.189683609Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:24:53.189694044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:24:53.189698397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:24:54.722393392Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:24:54.95761403Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35226"
level=debug ts=2026-04-21T04:24:56.094204181Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-21T04:24:58.190088885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:00.448981754Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=debug ts=2026-04-21T04:25:02.325387853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:25:03.189234903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:04.804146618Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:25:07.053168753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:25:08.189617756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:09.158425536Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-04-21T04:25:11.309784037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:25:11.318078031Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:25:12.187170397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:25:12.190891087Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:25:13.189613239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:13.513669629Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T04:25:15.216204268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:25:15.22167176Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T04:25:15.64252705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:44260"
level=debug ts=2026-04-21T04:25:17.868764103Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:25:18.189529098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:22.22344977Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:25:23.189672238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:25:23.189703507Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:25:23.189712715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:25:23.189752698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:25:24.960642286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56756"
level=debug ts=2026-04-21T04:25:26.46186451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:25:26.577591982Z caller=checkpoint.go:378 msg="writing series" size="400 kB"
level=debug ts=2026-04-21T04:25:26.577615822Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=18
level=warn ts=2026-04-21T04:25:28.189935469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:30.93996606Z caller=checkpoint.go:378 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-21T04:25:30.939994111Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.6 MB" series=1
level=warn ts=2026-04-21T04:25:33.189261944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:35.288090601Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
ts=2026-04-21T04:25:37.056360453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:25:38.190140093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:39.64273359Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-21T04:25:43.189871046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:43.997380745Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-04-21T04:25:48.189926366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:48.352336976Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-21T04:25:52.707003209Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:25:53.173100282Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:25:53.173596807Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:25:53.189961436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:25:53.189985985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:25:53.189975454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:25:53.190008975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:25:54.963673656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39674"
level=debug ts=2026-04-21T04:25:57.062302067Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:25:58.190004752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:25:58.778674305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:26:00.94753358Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:26:00.949418711Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:26:01.416987353Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:26:03.189747805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:26:05.77203384Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-04-21T04:26:05.790126894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:26:05.794287097Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T04:26:07.058182471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:26:08.1893716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:26:08.454474126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:26:08.459589597Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:26:10.126763586Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T04:26:10.177255917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:26:10.184793398Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:26:13.189481599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:26:14.481021554Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-21T04:26:18.189450751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:26:18.836179235Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-04-21T04:26:23.189232398Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:26:23.189263452Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:26:23.189271879Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:26:23.189338695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:26:23.190336855Z caller=checkpoint.go:390 msg="flushing series" totalSize="807 kB" series=11
level=info ts=2026-04-21T04:26:23.197926705Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-21T04:26:23.199278371Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008998763s
level=warn ts=2026-04-21T04:26:28.189444761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:26:31.507713556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:26:33.189492508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:26:35.536807176Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:26:37.059907254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:26:38.189817791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:26:43.18984954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:26:48.19024944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:26:49.544869847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:26:49.551397004Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:26:53.173067101Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:26:53.174182968Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:26:53.189443721Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T04:26:53.18947828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:26:53.189502759Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:26:53.189513106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:26:53.189517083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:26:53.189596277Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-21T04:26:53.189890744Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
ts=2026-04-21T04:26:54.970047774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48906"
level=debug ts=2026-04-21T04:26:57.545420012Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:26:58.190260997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:26:58.525637605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:26:58.530158798Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:26:58.530178774Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:26:58.53020145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:26:58.5335196Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:27:01.900510347Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=warn ts=2026-04-21T04:27:03.190036079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:27:04.847375582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:27:04.856280305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:27:06.213241596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:27:06.255349149Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:27:07.061880867Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:27:08.189812094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:27:08.592507702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:27:08.596461592Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:27:10.609817132Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-21T04:27:13.189748124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:27:14.151812202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40344"
level=debug ts=2026-04-21T04:27:14.964456424Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:27:18.189981873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:19.319288208Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-04-21T04:27:20.589114561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T04:27:23.189391797Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:27:23.189423412Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:27:23.189433249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:27:23.189432789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:23.673993203Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
ts=2026-04-21T04:27:24.972625396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39824"
level=debug ts=2026-04-21T04:27:28.029176093Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:27:28.189730845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:32.384030892Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=debug ts=2026-04-21T04:27:32.384054633Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-21T04:27:33.189382826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:27:34.121087264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:27:34.128716938Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:27:35.360284265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:27:35.366748077Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:27:36.345937446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:27:36.738753052Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-21T04:27:37.063900892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:27:38.189926739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:41.093993869Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=warn ts=2026-04-21T04:27:43.190130712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:27:44.154570027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42444"
level=debug ts=2026-04-21T04:27:45.448618394Z caller=checkpoint.go:378 msg="writing series" size="171 kB"
level=info ts=2026-04-21T04:27:46.493340931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:27:46.503420223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:27:48.190115189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:49.803414517Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=info ts=2026-04-21T04:27:53.173286198Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:27:53.173786635Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:27:53.189165687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:27:53.189191763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:27:53.189199946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:27:53.189202459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:54.158458724Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
ts=2026-04-21T04:27:54.975704931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:53114"
level=warn ts=2026-04-21T04:27:58.189387235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:27:58.513540958Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=debug ts=2026-04-21T04:28:02.867747191Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:28:03.190002134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:04.492388798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:28:07.066979699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:28:07.222492217Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:28:08.190032093Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:09.478720585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:28:11.577714108Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:28:13.189239205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:28:15.658997078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:38836"
level=debug ts=2026-04-21T04:28:15.932115706Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:28:18.189501765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:28:19.081082343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:28:19.08604553Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:28:19.66895654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:28:19.674199086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:28:19.674244318Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:28:19.674269369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:28:19.679363682Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:28:20.286772119Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-21T04:28:20.286797434Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T04:28:23.189163745Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:28:23.189197018Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:28:23.189223638Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:28:23.189197871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:24.642238802Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=warn ts=2026-04-21T04:28:28.19009763Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:28.997854404Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T04:28:28.99787629Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-04-21T04:28:32.77481158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:28:32.784773112Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:28:33.189275141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:33.351633014Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=debug ts=2026-04-21T04:28:35.080867815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:28:37.069795993Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:28:37.706680426Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:28:38.189859451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:42.061015718Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:28:43.189941719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:46.416739923Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=info ts=2026-04-21T04:28:46.495719004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:28:46.50708028Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:28:48.190021293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:50.771237937Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=debug ts=2026-04-21T04:28:53.172622515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:28:53.173669805Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:28:53.190128642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:28:53.190159307Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:28:53.190167672Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:28:53.190185686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:54.182573862Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:28:55.126366274Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T04:28:58.189979297Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:28:59.480303506Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-04-21T04:29:02.649817953Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:29:03.189986338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:03.835517889Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
ts=2026-04-21T04:29:07.072377754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:29:08.189673042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:08.18992373Z caller=checkpoint.go:378 msg="writing series" size="295 kB"
level=info ts=2026-04-21T04:29:10.382506427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:29:10.388908975Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:29:11.507903429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:29:11.512340458Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:29:12.545775457Z caller=checkpoint.go:378 msg="writing series" size="83 kB"
level=debug ts=2026-04-21T04:29:12.545800587Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-21T04:29:13.189429988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:29:15.663654517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:34264"
level=debug ts=2026-04-21T04:29:16.899550683Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:29:18.190060937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:21.25451845Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-21T04:29:23.189701592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:29:23.189714577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:29:23.18973724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:29:23.189746959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:29:25.609819827Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:29:28.189586847Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:29.221684052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:29:29.964765931Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-21T04:29:33.057952126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:29:33.059627855Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:29:33.189494286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:34.3198704Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:29:37.074436638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:29:38.189736488Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:38.674201509Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T04:29:39.408157335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:29:39.417136652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:29:39.417164101Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:29:39.417192777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:29:39.423449351Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:29:43.02955722Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:29:43.190005776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:29:44.165429817Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41672"
ts=2026-04-21T04:29:45.664923736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37694"
level=debug ts=2026-04-21T04:29:47.384050047Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:29:48.189780643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:29:51.158021158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:29:51.164987606Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:29:51.339028124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:29:51.346199536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:29:51.739005784Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-21T04:29:53.173427673Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:29:53.173844699Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:29:53.189260251Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:29:53.189296764Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:29:53.189305573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:29:53.189350084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:29:56.093568064Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-21T04:29:58.190042965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:00.448092332Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-21T04:30:02.043984904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:30:02.051989026Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:30:02.842552485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:30:03.189911295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:04.803136015Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:30:07.076731064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:30:08.190083085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:09.158025652Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-04-21T04:30:13.189515718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:13.512888067Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T04:30:14.168682797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60354"
level=debug ts=2026-04-21T04:30:15.162868094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T04:30:17.66951512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:30:17.675959028Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:30:17.867503955Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:30:18.18968617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:22.22245282Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:30:23.190007113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:30:23.190031702Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:30:23.190040147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:30:23.190037108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:26.577380162Z caller=checkpoint.go:378 msg="writing series" size="521 kB"
level=debug ts=2026-04-21T04:30:26.577401995Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=17
level=warn ts=2026-04-21T04:30:28.19023855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:30.379288295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:30:30.937308961Z caller=checkpoint.go:378 msg="writing series" size="8.3 MB"
level=debug ts=2026-04-21T04:30:30.937335008Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.3 MB" series=1
level=warn ts=2026-04-21T04:30:33.189762418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:35.287550994Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
ts=2026-04-21T04:30:37.079384375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:30:38.189238435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:39.642362793Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-21T04:30:43.19016363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:43.997159638Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
ts=2026-04-21T04:30:44.171882072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60878"
level=warn ts=2026-04-21T04:30:48.19009424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:48.351544071Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=info ts=2026-04-21T04:30:50.624991897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:30:50.633476712Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:30:52.340884719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:30:52.344983239Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:30:52.706092502Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:30:53.173236932Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:30:53.173253943Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=debug ts=2026-04-21T04:30:53.174502761Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T04:30:53.174543558Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974161 n=1
level=debug ts=2026-04-21T04:30:53.174584532Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974161 n=1
level=info ts=2026-04-21T04:30:53.18969885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:30:53.189716692Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:30:53.189727712Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:30:53.189725227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:30:53.443491857Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:30:57.061410869Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=debug ts=2026-04-21T04:30:57.766780255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:30:58.189255592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:31:01.416073484Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:31:03.189228149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:31:05.770512286Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
ts=2026-04-21T04:31:07.082491019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:31:08.190021333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:31:08.373311561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:31:08.380552761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:31:10.126161032Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=warn ts=2026-04-21T04:31:13.189473952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:31:14.480515112Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-21T04:31:18.190059741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:31:18.835803113Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=debug ts=2026-04-21T04:31:23.189997469Z caller=checkpoint.go:390 msg="flushing series" totalSize="906 kB" series=11
level=info ts=2026-04-21T04:31:23.190040818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:31:23.190067739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:31:23.190068577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:31:23.190082999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:31:23.195633584Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-21T04:31:23.197151291Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007417191s
ts=2026-04-21T04:31:24.993167971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:50682"
level=warn ts=2026-04-21T04:31:28.189544054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:31:29.46719817Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:31:30.834285389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:31:33.190124078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:31:35.786636264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:31:35.791939227Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T04:31:37.084024614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:31:38.189670889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:31:42.065322114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:31:42.072520974Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:31:43.189940603Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:31:48.189611044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:31:53.173549536Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:31:53.174550836Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:31:53.189898017Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T04:31:53.189924769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:31:53.189935339Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:31:53.189953724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:31:53.189963559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:31:53.190037259Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-21T04:31:53.190378315Z caller=checkpoint.go:378 msg="writing series" size="465 kB"
ts=2026-04-21T04:31:54.99574414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:34482"
level=debug ts=2026-04-21T04:31:56.055939558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:31:57.54584484Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:31:58.1900663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:01.900781311Z caller=checkpoint.go:378 msg="writing series" size="493 kB"
level=warn ts=2026-04-21T04:32:03.189507877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:32:04.126336784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:32:04.135996106Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:32:06.255565298Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:32:07.08616431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:32:08.190141822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:10.610636406Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=warn ts=2026-04-21T04:32:13.18932984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:32:14.177470581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57534"
level=debug ts=2026-04-21T04:32:14.965245452Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
ts=2026-04-21T04:32:15.677753284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42676"
level=debug ts=2026-04-21T04:32:16.575097856Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:32:18.190187711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:19.320332835Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=debug ts=2026-04-21T04:32:19.320366561Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-21T04:32:23.189662461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:32:23.189693223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:32:23.18970284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:32:23.18969716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:23.674989331Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-21T04:32:28.029497226Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:32:28.189857498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:29.288480511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:32:32.384799958Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:32:33.190163651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:32:33.961123013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:32:33.972044524Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:32:36.020697598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:32:36.025859506Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:32:36.738826217Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-21T04:32:37.088251372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:32:38.189608287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:41.09447244Z caller=checkpoint.go:378 msg="writing series" size="170 kB"
level=warn ts=2026-04-21T04:32:43.190194955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:45.449262126Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-21T04:32:48.19030097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:49.80412948Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=info ts=2026-04-21T04:32:53.173323853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:32:53.174277782Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:32:53.189715894Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:32:53.189736404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:32:53.189744719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:32:53.189760867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:54.158770452Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-04-21T04:32:56.583802869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:32:56.594403667Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:32:58.189677561Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:32:58.513295168Z caller=checkpoint.go:378 msg="writing series" size="375 kB"
level=debug ts=2026-04-21T04:32:58.513322499Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-21T04:32:59.599594001Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:33:02.868281104Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:33:03.189430484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:06.731774204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:33:06.733570547Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-21T04:33:07.089998506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:33:07.223267884Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:33:08.189640476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:11.577703109Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:33:13.189959712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:15.932369249Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:33:18.189677161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:20.287412605Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=warn ts=2026-04-21T04:33:23.190018321Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:33:23.19002342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:33:23.190043246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:33:23.190052417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:33:24.642560654Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
ts=2026-04-21T04:33:25.003113232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37114"
level=info ts=2026-04-21T04:33:25.735430324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:33:25.742676365Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:33:28.189810713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:33:28.468921874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:33:28.473342651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:33:28.473369252Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:33:28.473396776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:33:28.476561545Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:33:28.998910642Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-21T04:33:28.99893329Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-04-21T04:33:33.189413294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:33.351907864Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-04-21T04:33:33.363026882Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:33:37.09277078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:33:37.706843275Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:33:38.18962917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:33:39.306374692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:33:39.318367071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:33:40.872196766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:33:40.878941389Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:33:42.061346993Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=debug ts=2026-04-21T04:33:42.281251412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:33:43.190101927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:46.416101043Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:33:48.190079681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:50.771056465Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-21T04:33:53.17380402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:33:53.175099952Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:33:53.1893952Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:33:53.189422627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:33:53.189432528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:33:53.18949417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:33:55.006133003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:55960"
level=debug ts=2026-04-21T04:33:55.12684876Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
level=warn ts=2026-04-21T04:33:58.189354191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:33:59.480987523Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=warn ts=2026-04-21T04:34:03.189878097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:03.836030304Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=debug ts=2026-04-21T04:34:07.016317004Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:34:07.095753105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:34:08.189717092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:08.190955568Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=debug ts=2026-04-21T04:34:08.19097296Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-21T04:34:10.134257433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:34:10.143099188Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:34:12.545514307Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-04-21T04:34:13.189295288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:16.900443403Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:34:18.190291632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:21.255688932Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:34:22.991961159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:34:22.999588037Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:34:23.189778887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:34:23.189812987Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:34:23.189803338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:34:23.189824183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:34:25.609873475Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:34:28.189689543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:29.965152595Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T04:34:32.799199331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:34:32.806731877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:34:33.190027756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:34.319808301Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:34:37.098412079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:34:38.190192089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:38.674784808Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:34:39.050591558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:34:39.215373351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:34:43.029006539Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:34:43.189597833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:34:45.690093031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:58754"
level=debug ts=2026-04-21T04:34:47.383816037Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:34:48.189964678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:34:51.738721541Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=info ts=2026-04-21T04:34:53.173036808Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:34:53.174025561Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:34:53.189306638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:34:53.189314128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:34:53.1893324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:34:53.189340578Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:34:56.094333315Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=warn ts=2026-04-21T04:34:58.189704315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:34:59.043341203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:34:59.05097671Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:35:00.449303223Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:35:03.189651923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:04.803683855Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:35:07.101036037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:35:08.189580759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:09.158331258Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=debug ts=2026-04-21T04:35:12.689795327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:35:13.189394708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:13.513603311Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-21T04:35:17.868311318Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:35:18.189431517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:35:20.293822196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:35:20.300674452Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:35:20.880338215Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:35:22.222587891Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:35:22.458287813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:35:22.463564139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:35:22.463589027Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:35:22.463617462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:35:22.468977413Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:35:23.189981168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:35:23.190013134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:35:23.190022393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:35:23.190026211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:26.578359535Z caller=checkpoint.go:378 msg="writing series" size="622 kB"
level=debug ts=2026-04-21T04:35:26.578383917Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=18
level=warn ts=2026-04-21T04:35:28.190163049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:30.940152205Z caller=checkpoint.go:378 msg="writing series" size="9.1 MB"
level=debug ts=2026-04-21T04:35:30.940178197Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.1 MB" series=1
level=warn ts=2026-04-21T04:35:33.189310896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:35.288109888Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
ts=2026-04-21T04:35:37.104640827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:35:38.190246895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:39.642607664Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-04-21T04:35:41.363048088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:35:41.370092384Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:35:42.059163202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:35:43.189503654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:43.997306942Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
ts=2026-04-21T04:35:45.694890395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:57470"
level=warn ts=2026-04-21T04:35:48.189279265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:35:48.351662213Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-04-21T04:35:51.527536658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:35:51.53338268Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:35:52.706743606Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-21T04:35:53.172965102Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:35:53.174036173Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T04:35:53.189408618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:35:53.189409092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:35:53.189436558Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:35:53.189447209Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:35:57.061762294Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:35:58.190148186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:36:01.416399619Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:36:03.18942415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:36:03.625576874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:36:03.634077005Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:36:05.771365505Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
ts=2026-04-21T04:36:07.108041963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:36:08.189862645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:36:08.554095018Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:36:10.125771763Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=warn ts=2026-04-21T04:36:13.189917625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:36:13.992394216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:36:14.480983417Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-04-21T04:36:16.247081925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:36:16.261766562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:36:16.26179414Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:36:16.26182642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:36:16.268889988Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:36:18.189821209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:36:18.835720147Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-04-21T04:36:23.189740316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:36:23.18977138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:36:23.189783065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:36:23.189776085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:36:23.190858901Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=11
level=info ts=2026-04-21T04:36:23.19676824Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-21T04:36:23.198265249Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008096215s
level=info ts=2026-04-21T04:36:28.107369817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:36:28.116703869Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:36:28.189396874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:36:33.189262743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:36:37.110467044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:36:38.190018758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:36:39.218385993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:36:43.189770499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:36:44.197546304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49562"
level=warn ts=2026-04-21T04:36:48.189675467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:36:50.292106507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:36:50.297331422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:36:53.173060371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:36:53.173300169Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:36:53.189823246Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-21T04:36:53.189853893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:36:53.189853388Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:36:53.189874108Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:36:53.18988274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:36:53.190004977Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-04-21T04:36:53.190413221Z caller=checkpoint.go:378 msg="writing series" size="485 kB"
level=info ts=2026-04-21T04:36:54.268776264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:36:54.274569106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:36:56.541115094Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:36:56.542972091Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:36:57.545950274Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:36:58.189821972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:01.900323495Z caller=checkpoint.go:378 msg="writing series" size="506 kB"
level=warn ts=2026-04-21T04:37:03.189751577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:06.255523252Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-21T04:37:07.111970544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T04:37:07.553283833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:37:07.562840084Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:37:08.190144399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:10.609826362Z caller=checkpoint.go:378 msg="writing series" size="9.3 kB"
level=warn ts=2026-04-21T04:37:13.189993504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:37:14.2002435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58664"
level=debug ts=2026-04-21T04:37:14.964712059Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-21T04:37:14.968986811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:37:18.189673432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:19.32016679Z caller=checkpoint.go:378 msg="writing series" size="106 kB"
level=debug ts=2026-04-21T04:37:19.32019651Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-21T04:37:23.190051135Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:37:23.190085264Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:37:23.19009635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:37:23.190095599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:23.675094819Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-21T04:37:28.029203052Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:37:28.189450847Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:37:32.305240694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:37:32.308984481Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:37:32.38436651Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:37:33.190037869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:33.865759203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T04:37:35.624661512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:37:35.629912461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:37:36.739374517Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
ts=2026-04-21T04:37:37.113921746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:37:38.189527639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:39.306865464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:37:41.093610152Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-21T04:37:43.189479884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:45.448357766Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
ts=2026-04-21T04:37:45.706734711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:51386"
level=warn ts=2026-04-21T04:37:48.189429442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:49.804173111Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=info ts=2026-04-21T04:37:53.17322142Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:37:53.174147602Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:37:53.189479446Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:37:53.189510825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:37:53.189519614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:37:53.189539467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:54.158043366Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
ts=2026-04-21T04:37:55.021417134Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48254"
level=info ts=2026-04-21T04:37:56.394771882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:37:56.402528136Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:37:58.18951092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:37:58.513851419Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=debug ts=2026-04-21T04:37:58.513879022Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-04-21T04:38:02.868237099Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T04:38:03.190091224Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:38:07.116457679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:38:07.223544642Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:38:08.189450898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:11.031437245Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:38:11.578128843Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-21T04:38:13.190168796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:13.908097681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:38:15.932368746Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:38:18.190156992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:38:20.082632213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:38:20.086305554Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:38:20.287193541Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:38:21.832709181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:38:21.837659283Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:38:23.1891872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:38:23.189241829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:38:23.189250422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:38:23.1892397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:24.642963741Z caller=checkpoint.go:378 msg="writing series" size="257 kB"
level=warn ts=2026-04-21T04:38:28.189669378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:28.998860764Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-21T04:38:28.998883846Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-04-21T04:38:33.189748454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:33.352238017Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
ts=2026-04-21T04:38:37.118337094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:38:37.706710531Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:38:38.189666771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:41.368417583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:38:42.061698325Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:38:43.189433284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:38:44.208270763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45550"
level=debug ts=2026-04-21T04:38:46.416609417Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-21T04:38:48.189994331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:50.771667373Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-21T04:38:51.417857113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:38:51.433510846Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:38:53.173102695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:38:53.173586371Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:38:53.190045246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:38:53.190080562Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:38:53.190088779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:38:53.190106063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:55.126633344Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=warn ts=2026-04-21T04:38:58.189866885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:38:59.480955481Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=warn ts=2026-04-21T04:39:03.189389259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:39:03.584317745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:39:03.591313339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:39:03.83582481Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-04-21T04:39:06.061394482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:39:06.065356206Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:39:06.770087372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:39:07.1207946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:39:08.189987535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:39:08.191321406Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
level=debug ts=2026-04-21T04:39:08.191339238Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-21T04:39:11.596139141Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:39:12.545758126Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-04-21T04:39:13.189862163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:39:14.211187369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37908"
level=debug ts=2026-04-21T04:39:16.900697885Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:39:18.189667001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:39:21.255701624Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:39:23.189301011Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:39:23.189333141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:39:23.18934134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:39:23.189392696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:39:25.610109026Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:39:28.189256555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:39:29.965879825Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:39:33.189881787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:39:34.320017006Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:39:37.123059147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:39:38.189726686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:39:38.256907894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:39:38.266021078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:39:38.674317389Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:39:43.029549031Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:39:43.189695905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:39:44.858805476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:39:45.977301931Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:39:47.384443823Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:39:48.189825228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:39:51.126855879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:39:51.132224718Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:39:51.132248633Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:39:51.132273999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:39:51.135838145Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:39:51.517934169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:39:51.524056102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:39:51.739446955Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=info ts=2026-04-21T04:39:53.173192931Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:39:53.17318889Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:39:53.189684943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:39:53.189719839Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:39:53.189731834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:39:53.189748819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:39:55.029709959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59386"
level=debug ts=2026-04-21T04:39:56.093929528Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T04:39:58.189860287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:00.449178605Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-21T04:40:02.020034965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:40:02.027921326Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:40:03.189672652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:04.803756671Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:40:07.124672466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:40:08.189241804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:09.158502569Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-04-21T04:40:13.189245624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:13.513459262Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T04:40:14.214796726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55180"
ts=2026-04-21T04:40:15.720275143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59888"
level=debug ts=2026-04-21T04:40:17.868389793Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:40:18.18955712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:20.215701905Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:40:22.223426836Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=warn ts=2026-04-21T04:40:23.189657267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:40:23.189667587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:40:23.189689213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:40:23.189697835Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:40:26.577748175Z caller=checkpoint.go:378 msg="writing series" size="536 kB"
level=debug ts=2026-04-21T04:40:26.577771809Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=18
level=info ts=2026-04-21T04:40:27.836138985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:40:27.846484136Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:40:28.190218216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:29.992006838Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:40:29.993634786Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:40:30.939773849Z caller=checkpoint.go:378 msg="writing series" size="9.6 MB"
level=debug ts=2026-04-21T04:40:30.939798375Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.6 MB" series=1
level=info ts=2026-04-21T04:40:33.072798335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:40:33.077759525Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:40:33.189248076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:35.28734857Z caller=checkpoint.go:378 msg="writing series" size="243 kB"
ts=2026-04-21T04:40:37.127487755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:40:38.189452902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:39.642682801Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-21T04:40:40.25573488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:40:40.261397192Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:40:43.190031499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:43.997240302Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-04-21T04:40:45.732270694Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:40:48.189737309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:48.352158192Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-04-21T04:40:52.707188688Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:40:53.173204812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:40:53.174376351Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:40:53.189719426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:40:53.189746508Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:40:53.189755459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:40:53.1897503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:40:57.061173015Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:40:58.18940776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:41:01.416524622Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:41:03.189751874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:41:05.771711919Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
ts=2026-04-21T04:41:07.131123607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:41:08.190168157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:41:10.126583126Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T04:41:10.22416703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:41:10.23617045Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:41:13.189497127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:41:13.781273518Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:41:14.219423216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48374"
level=debug ts=2026-04-21T04:41:14.481469976Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=debug ts=2026-04-21T04:41:14.481502553Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-21T04:41:18.18994717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:41:18.836292758Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=info ts=2026-04-21T04:41:23.189202604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:41:23.189265897Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:41:23.189278102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:41:23.189286407Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:41:23.190192981Z caller=checkpoint.go:390 msg="flushing series" totalSize="58 kB" series=1
level=info ts=2026-04-21T04:41:23.196281168Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-21T04:41:23.197960707Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00780809s
level=info ts=2026-04-21T04:41:25.74145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:41:25.745679766Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:41:25.756679676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T04:41:25.866179607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:41:25.871773048Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:41:25.871799803Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:41:25.871830683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:41:25.877996609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:41:28.189428991Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:41:33.189883057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:41:37.133822054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:41:38.189787432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:41:43.189239618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:41:44.222086143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47580"
level=debug ts=2026-04-21T04:41:44.395000346Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:41:44.395065716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:41:44.402188135Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T04:41:45.589697502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:41:45.595588118Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:41:48.190097702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:41:53.173190121Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:41:53.173578915Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:41:53.189022386Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T04:41:53.189163504Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-21T04:41:53.189256444Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:41:53.189285288Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:41:53.189295799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:41:53.189308761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:41:53.189518894Z caller=checkpoint.go:378 msg="writing series" size="287 kB"
ts=2026-04-21T04:41:55.039031886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:51824"
level=debug ts=2026-04-21T04:41:57.544971111Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:41:58.190095291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:42:00.61512201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:42:00.620246374Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:42:01.899975317Z caller=checkpoint.go:378 msg="writing series" size="530 kB"
level=warn ts=2026-04-21T04:42:03.189748131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:42:06.172070679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:42:06.180372562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:42:06.254502945Z caller=checkpoint.go:378 msg="writing series" size="639 B"
ts=2026-04-21T04:42:07.135124394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:42:08.189470135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:10.609452061Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T04:42:11.711201936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:42:11.717504418Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:42:13.189518162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:42:14.22466969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34896"
level=debug ts=2026-04-21T04:42:14.963875174Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-21T04:42:15.172722199Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:42:15.32914823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:42:15.334883747Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:42:18.189377664Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:19.319567865Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-21T04:42:23.18958064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:42:23.189581023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:23.189608349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:42:23.189616936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:42:23.67371555Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-21T04:42:27.557498522Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:42:28.028953425Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-21T04:42:28.028978115Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-21T04:42:28.189358771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:32.383698561Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:42:33.189573863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:36.738424583Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-21T04:42:37.136706914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:42:38.190148617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:42:39.817567522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:42:39.823531122Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:42:41.09376583Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=warn ts=2026-04-21T04:42:43.190158515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:45.447639723Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
ts=2026-04-21T04:42:45.734848324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37688"
level=warn ts=2026-04-21T04:42:48.189933104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:48.632075085Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:42:49.802645017Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=info ts=2026-04-21T04:42:50.217838884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:42:50.229930496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:42:50.229961409Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:42:50.230000353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:42:50.236778867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:42:53.172944015Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:42:53.17400906Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:42:53.189409555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:42:53.189440683Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:42:53.18944968Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:42:53.189453801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:54.157332274Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-04-21T04:42:54.694261227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:42:54.699432273Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:42:58.189395034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:42:58.513068085Z caller=checkpoint.go:378 msg="writing series" size="460 kB"
level=debug ts=2026-04-21T04:42:58.513090638Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-04-21T04:43:01.784507614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:43:01.791667346Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:43:02.867659971Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T04:43:03.189999344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:43:03.632000544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:43:03.63673963Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T04:43:07.13894866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:43:07.222230585Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:43:08.189505085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:11.57685012Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T04:43:13.189746352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:15.931939388Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:43:18.189251655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:20.252981262Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:43:20.286261549Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-21T04:43:22.93959678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T04:43:23.189718079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:43:23.189748644Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:43:23.18975679Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:43:23.18975223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:24.641549241Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=warn ts=2026-04-21T04:43:28.189336633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:28.997451707Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T04:43:28.997473198Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=warn ts=2026-04-21T04:43:33.189476606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:33.351193394Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
ts=2026-04-21T04:43:37.141834957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=info ts=2026-04-21T04:43:37.647242348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:43:37.656607166Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:43:37.70584815Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-21T04:43:37.977854675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:43:37.98247556Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:43:38.189324001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:42.061164974Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:43:43.189922523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:43:44.333828799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:43:44.338053332Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:43:45.163325771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:43:45.16871575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:43:46.415972474Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=warn ts=2026-04-21T04:43:48.189426046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:50.770665208Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=debug ts=2026-04-21T04:43:52.700956979Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:43:53.173058705Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:43:53.173062799Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:43:53.189467391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:43:53.189493546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:43:53.189502139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:43:53.189514419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:55.126087522Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=warn ts=2026-04-21T04:43:58.190090786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:43:59.480777425Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:44:03.189836301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:03.700844979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:44:03.83500485Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:44:07.145003577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:44:08.190227527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:08.190354778Z caller=checkpoint.go:378 msg="writing series" size="301 kB"
level=debug ts=2026-04-21T04:44:08.190372466Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-04-21T04:44:12.544420285Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=warn ts=2026-04-21T04:44:13.189306733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:44:15.743015924Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:47196"
level=debug ts=2026-04-21T04:44:16.89991966Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:44:18.18991737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:44:19.815846515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:44:19.824506025Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:44:21.254944302Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:44:23.189967521Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:44:23.18999835Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:44:23.190007332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:44:23.189996758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:44:25.197469988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:44:25.202073499Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:44:25.58126557Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:44:25.609374326Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:44:28.189941798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:29.963763217Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:44:33.189779887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:34.319059113Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:44:37.147187934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:44:38.189968282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:38.673416378Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T04:44:39.046334249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:44:39.054919732Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:44:41.517072976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:44:41.523352232Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:44:43.029063419Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:44:43.189583566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:44:44.237067919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39446"
ts=2026-04-21T04:44:45.744678547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:43436"
level=debug ts=2026-04-21T04:44:47.383028038Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=debug ts=2026-04-21T04:44:47.491794345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:44:47.493603601Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:44:48.189585084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:51.738438051Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=debug ts=2026-04-21T04:44:53.17275939Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:44:53.173889414Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T04:44:53.18928293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:44:53.189283281Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:44:53.189306698Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:44:53.189316127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:44:56.093024715Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=warn ts=2026-04-21T04:44:58.189460525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:44:59.099565591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:45:00.447811251Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:45:03.189274279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:04.802542509Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:45:07.148159525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:45:08.189362869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:09.157502812Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-21T04:45:13.190069417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:13.512222597Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T04:45:14.239797203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:36246"
ts=2026-04-21T04:45:15.745774741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:48580"
level=info ts=2026-04-21T04:45:16.51217989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:45:16.519951576Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:45:17.867199765Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:45:18.18955028Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:45:19.244962381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:45:19.24950834Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:45:22.222764127Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:45:23.189871986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:45:23.189888137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:23.189903539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:45:23.189911821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:45:26.577254737Z caller=checkpoint.go:378 msg="writing series" size="525 kB"
level=debug ts=2026-04-21T04:45:26.57727808Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=18
level=warn ts=2026-04-21T04:45:28.18999474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:45:30.200642619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:45:30.207765413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:45:30.940835555Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-21T04:45:30.94085717Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=1
level=info ts=2026-04-21T04:45:31.655585933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:45:31.66166591Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:45:32.62172759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:45:33.189571495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:35.28717458Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
ts=2026-04-21T04:45:37.150770935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:45:38.190054656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:39.641721748Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-21T04:45:43.189797593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:43.996336285Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=debug ts=2026-04-21T04:45:44.765509413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:45:48.189965326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:45:48.351412227Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=debug ts=2026-04-21T04:45:52.705915779Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:45:53.173014222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:45:53.173334189Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=debug ts=2026-04-21T04:45:53.174575296Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T04:45:53.174615848Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974162 n=1
level=debug ts=2026-04-21T04:45:53.174656064Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974162 n=1
level=info ts=2026-04-21T04:45:53.189808806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:45:53.18982359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:45:53.189818325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:45:53.189832191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:45:57.061398368Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:45:58.189369031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:46:01.415631847Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:46:03.190004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:46:05.770516935Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
ts=2026-04-21T04:46:07.154070856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:46:07.192057328Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:46:08.189985178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:46:09.633291173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:46:09.641922843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:46:10.125339375Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=debug ts=2026-04-21T04:46:10.12536606Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-21T04:46:10.21987443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:46:10.225809523Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:46:10.22583463Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:46:10.225863965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:46:10.230018411Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:46:13.189385211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:46:13.603121399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:46:13.60770596Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:46:14.479935803Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-21T04:46:18.189406611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:46:18.835230324Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=info ts=2026-04-21T04:46:22.121004139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:46:22.127192325Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:46:23.189277776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:46:23.189279402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:46:23.189306825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:46:23.189315525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:46:23.190360847Z caller=checkpoint.go:390 msg="flushing series" totalSize="125 kB" series=2
level=info ts=2026-04-21T04:46:23.194492248Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-21T04:46:23.196002833Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006660902s
level=info ts=2026-04-21T04:46:27.515179538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:46:27.519506909Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:46:28.189783954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:46:33.189544906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:46:33.413550498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:46:37.156685159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:46:37.279492127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:46:38.189480795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:46:43.189680019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:46:48.189231554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:46:48.238364934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:46:48.243847045Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:46:51.051096926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:46:51.058491092Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:46:53.173246251Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:46:53.173246756Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:46:53.189734799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T04:46:53.18974843Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-21T04:46:53.189767873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:46:53.189777035Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:46:53.189818316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:46:53.189921782Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-21T04:46:53.190595391Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=info ts=2026-04-21T04:46:55.842827769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:46:55.848120511Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:46:57.545917558Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:46:58.189619363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:01.901040733Z caller=checkpoint.go:378 msg="writing series" size="594 kB"
level=debug ts=2026-04-21T04:47:02.823393982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:47:03.189428607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:06.254875288Z caller=checkpoint.go:378 msg="writing series" size="639 B"
ts=2026-04-21T04:47:07.159401575Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:47:08.189249724Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:47:08.937993792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:47:08.946856555Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:47:10.610294594Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-21T04:47:13.189499739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:47:14.250224302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39674"
level=debug ts=2026-04-21T04:47:14.964887233Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-21T04:47:18.190045392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:19.320079705Z caller=checkpoint.go:378 msg="writing series" size="119 kB"
level=debug ts=2026-04-21T04:47:19.320106777Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-21T04:47:23.190033079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:47:23.190043624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:23.190059996Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:47:23.190068536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:47:23.674586402Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=debug ts=2026-04-21T04:47:28.029348715Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:47:28.18933733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:29.724323741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:47:31.945582023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:47:31.951615508Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:47:32.384259365Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:47:33.190061505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:47:35.688092014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:47:35.695575674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:47:36.738842446Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-21T04:47:37.162097407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:47:37.286665516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:47:38.189493684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:41.094054987Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-21T04:47:43.189955134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:47:44.012594859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:47:44.018909163Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T04:47:44.253702866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41866"
level=debug ts=2026-04-21T04:47:45.448575556Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=warn ts=2026-04-21T04:47:48.189437356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:49.804467754Z caller=checkpoint.go:378 msg="writing series" size="374 kB"
level=info ts=2026-04-21T04:47:53.173008803Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:47:53.173014314Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:47:53.189415044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:47:53.189427647Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:47:53.189446978Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:47:53.18945534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:47:54.158978361Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=debug ts=2026-04-21T04:47:54.159004934Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-21T04:47:58.18937385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:47:58.51355826Z caller=checkpoint.go:378 msg="writing series" size="329 kB"
level=info ts=2026-04-21T04:48:01.006407203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:48:01.012260315Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:48:01.01228372Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:48:01.012311814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:48:01.01724902Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:48:02.633077906Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:48:02.867876693Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T04:48:03.189976498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:48:07.165442611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:48:07.223087701Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:48:08.189973709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:11.578242323Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T04:48:13.19002367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:48:14.256667554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45336"
ts=2026-04-21T04:48:15.762204306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:54822"
level=debug ts=2026-04-21T04:48:15.932993969Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:48:18.189286738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:48:18.633348118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:48:18.641837425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:48:20.287486843Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=warn ts=2026-04-21T04:48:23.189461464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:48:23.189487584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:48:23.18951177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:48:23.189522331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:48:23.772862261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:48:23.777097645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:48:24.642506171Z caller=checkpoint.go:378 msg="writing series" size="291 kB"
level=info ts=2026-04-21T04:48:24.907174217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:48:24.91458763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:48:26.375277287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:48:26.379344741Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:48:26.379368062Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:48:26.379394736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:48:26.382752842Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:48:28.189711734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:28.998574561Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-21T04:48:28.998595718Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=8
level=warn ts=2026-04-21T04:48:33.189737155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:33.352117105Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-04-21T04:48:35.664920928Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:48:37.168747426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:48:37.7067131Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:48:38.189779897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:38.33018616Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:48:42.061531628Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:48:43.189316206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:48:46.01290479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:48:46.018987354Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:48:46.416311666Z caller=checkpoint.go:378 msg="writing series" size="245 kB"
level=warn ts=2026-04-21T04:48:48.189605863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:50.771610196Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-21T04:48:50.834746315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:48:50.840475087Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:48:53.17301229Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:48:53.173474368Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:48:53.189759345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:48:53.18977862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:48:53.189786532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:48:53.189783884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:55.126475829Z caller=checkpoint.go:378 msg="writing series" size="333 kB"
level=warn ts=2026-04-21T04:48:58.189595932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:48:59.480986621Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-21T04:49:03.190025643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:03.835692897Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=debug ts=2026-04-21T04:49:06.721050114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:49:07.171288743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:49:08.189482382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:08.190723809Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=debug ts=2026-04-21T04:49:08.190739634Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-04-21T04:49:09.469008041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:49:09.474588577Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:49:12.545464272Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:49:13.190126012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:16.900286886Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-21T04:49:17.904767233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:49:17.909420848Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:49:18.189387155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:49:19.575613667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:49:19.584245546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:49:19.584269579Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:49:19.584298799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:49:19.591419614Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:49:21.255650159Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-21T04:49:23.190150505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:49:23.190160028Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:49:23.190181011Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:49:23.190189873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:49:25.068025794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:39234"
level=debug ts=2026-04-21T04:49:25.61012026Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:49:28.190033902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:29.964905556Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-21T04:49:31.831874414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:49:31.833618285Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T04:49:33.190171944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:49:33.673920695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:49:33.680196802Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:49:34.319364126Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-04-21T04:49:36.842939138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:49:36.850401851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:49:37.173878443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:49:37.636414622Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:49:38.189695833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:38.674289148Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:49:43.028965437Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:49:43.189743435Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:47.384221984Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:49:48.189583675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:49:51.738862147Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-04-21T04:49:52.891273444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:49:52.900045152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T04:49:53.173486423Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:49:53.173966394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:49:53.189259626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:49:53.189281842Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:49:53.18930364Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:49:53.189312366Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:49:56.094432129Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=warn ts=2026-04-21T04:49:58.189757378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:50:00.157342331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:50:00.164355682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:50:00.44928162Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-21T04:50:00.819471496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:50:00.823871521Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:50:02.457738735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:50:03.189337658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:04.803106333Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T04:50:07.176243796Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:50:08.189248398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:09.158445918Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=debug ts=2026-04-21T04:50:10.008235457Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:50:13.189297472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:13.513359393Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-21T04:50:17.868364411Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:50:18.189426717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:22.222524031Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T04:50:22.240581516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:50:22.246586815Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T04:50:23.190139064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:50:23.190166207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:50:23.190173119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:50:23.190174937Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:50:26.577703678Z caller=checkpoint.go:378 msg="writing series" size="614 kB"
level=debug ts=2026-04-21T04:50:26.577725788Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=18
level=warn ts=2026-04-21T04:50:28.189447762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:30.945251624Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-21T04:50:30.945278115Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-04-21T04:50:33.189935522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:35.287241383Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
ts=2026-04-21T04:50:37.177755784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:50:38.176381959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:50:38.18949237Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:39.642224615Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=info ts=2026-04-21T04:50:40.512872911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:50:40.518586925Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:50:43.190105306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:43.997497674Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
ts=2026-04-21T04:50:45.77653635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:38630"
level=warn ts=2026-04-21T04:50:48.18935845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:48.351830005Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=info ts=2026-04-21T04:50:52.151725871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:50:52.158610359Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:50:52.706288238Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:50:53.173023486Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:50:53.173344304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:50:53.189711988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:50:53.189710935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:50:53.189737855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:50:53.189746361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:50:54.105895235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:50:54.112931992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:50:57.061936703Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:50:58.189953534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:51:01.416957025Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:51:03.189889179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:51:05.771078292Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
ts=2026-04-21T04:51:07.179352691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:51:08.19011663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:51:10.125942749Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T04:51:10.903260645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:51:10.908476693Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:51:11.384973086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:51:13.189425536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:51:13.653509875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:51:14.481424449Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-21T04:51:18.189694405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:51:18.836399627Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-21T04:51:23.189226812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:51:23.189236115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:51:23.189259328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:51:23.189269152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:51:23.190344613Z caller=checkpoint.go:390 msg="flushing series" totalSize="850 kB" series=11
level=info ts=2026-04-21T04:51:23.198020608Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-21T04:51:23.199777192Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009695185s
ts=2026-04-21T04:51:25.07696359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41654"
level=warn ts=2026-04-21T04:51:28.190011888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:51:33.189340528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:51:35.893271378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:51:35.902585737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:51:37.181122541Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=info ts=2026-04-21T04:51:38.178078277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:51:38.182333704Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:51:38.189389833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T04:51:43.189237808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:51:44.273344197Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:53312"
level=debug ts=2026-04-21T04:51:45.443278635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:51:45.780786188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:36776"
level=warn ts=2026-04-21T04:51:48.189488091Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:51:48.980395022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:51:48.987553348Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T04:51:52.782615067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:51:52.788912173Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:51:53.172722048Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:51:53.173764875Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:51:53.189052978Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T04:51:53.189242746Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-21T04:51:53.189260549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:51:53.189283877Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:51:53.189292346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:51:53.189290312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:51:53.189576474Z caller=checkpoint.go:378 msg="writing series" size="375 kB"
level=debug ts=2026-04-21T04:51:57.545178017Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:51:58.190145762Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:01.899597605Z caller=checkpoint.go:378 msg="writing series" size="629 kB"
level=warn ts=2026-04-21T04:52:03.189934319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:06.254705701Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T04:52:07.183936438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:52:08.189762661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:08.812035657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:52:10.608864256Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-21T04:52:12.993588259Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:52:13.189627918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:14.964070186Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-21T04:52:18.190262039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:19.318736969Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=debug ts=2026-04-21T04:52:19.318761469Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-04-21T04:52:23.189233944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:52:23.18925607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:52:23.189279169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:52:23.189287825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:52:23.674369184Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T04:52:25.540671685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:52:25.553620682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:52:28.02906036Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:52:28.189378988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:32.383317196Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T04:52:33.190227393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:52:35.248929738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:52:35.253161225Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:52:36.738076546Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-21T04:52:37.187502481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=info ts=2026-04-21T04:52:37.396512244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:52:37.40113766Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:52:37.401161847Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:52:37.401189203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:52:37.404671309Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:52:38.190120467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:39.70032675Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:52:41.09356554Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=warn ts=2026-04-21T04:52:43.189417611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:52:44.277328341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:43614"
level=debug ts=2026-04-21T04:52:44.703247118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:52:45.4485764Z caller=checkpoint.go:378 msg="writing series" size="215 kB"
level=warn ts=2026-04-21T04:52:48.189690824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:49.802753185Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=info ts=2026-04-21T04:52:51.85077293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:52:51.857777039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:52:53.172727427Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:52:53.17379056Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:52:53.19008085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:52:53.1901044Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:52:53.190113855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:52:53.190164922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:54.157669979Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-21T04:52:55.097497126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:52:55.101241393Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:52:58.189738011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:52:58.513311283Z caller=checkpoint.go:378 msg="writing series" size="459 kB"
level=debug ts=2026-04-21T04:52:58.51333625Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-21T04:53:02.867802398Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T04:53:03.189259253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:53:07.189949164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:53:07.22271266Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=warn ts=2026-04-21T04:53:08.189680949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:11.576943117Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T04:53:13.190134335Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:15.390596842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:53:15.932133853Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:53:18.189458234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:53:18.456338056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:53:18.463704735Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:53:20.286427358Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:53:23.189690294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:53:23.189699367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:23.189714802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:53:23.189723575Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:53:24.641930879Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
ts=2026-04-21T04:53:25.088199112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46018"
level=info ts=2026-04-21T04:53:25.516423443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:53:25.5241717Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:53:28.190133306Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:28.997587968Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-21T04:53:28.997611204Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=7
level=info ts=2026-04-21T04:53:32.605527609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:53:32.610239134Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:53:33.18930919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:33.351843565Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-04-21T04:53:34.040457076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:53:34.04594897Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T04:53:37.191797662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:53:37.70623315Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:53:38.189255601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:42.060527582Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:53:43.190097017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:53:44.283352765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59236"
level=debug ts=2026-04-21T04:53:45.751835706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:53:45.753441176Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T04:53:46.416065328Z caller=checkpoint.go:378 msg="writing series" size="288 kB"
level=warn ts=2026-04-21T04:53:48.190196006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:50.770778534Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-21T04:53:51.14588307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:53:53.173015137Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:53:53.173468125Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:53:53.189786149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:53:53.18981303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:53:53.189839767Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:53:53.189848629Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T04:53:55.091621217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:41598"
level=debug ts=2026-04-21T04:53:55.12538875Z caller=checkpoint.go:378 msg="writing series" size="412 kB"
level=warn ts=2026-04-21T04:53:58.18990079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:53:59.480673809Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=info ts=2026-04-21T04:54:00.76431574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:54:00.770950232Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:54:03.189349579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:03.834985493Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
ts=2026-04-21T04:54:07.19434479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:54:08.189877851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:08.190160429Z caller=checkpoint.go:378 msg="writing series" size="447 kB"
level=debug ts=2026-04-21T04:54:08.190175952Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-04-21T04:54:12.487342018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:54:12.496291108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:54:12.544434133Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=warn ts=2026-04-21T04:54:13.189581515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:54:16.004670442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:54:16.010768341Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:54:16.010788043Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:54:16.010813111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:54:16.016459336Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:54:16.315578224Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:54:16.899361842Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:54:18.189597375Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:21.25466449Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-21T04:54:23.189722416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:54:23.189727438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:54:23.189747624Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:54:23.189756016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:54:25.60936468Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-21T04:54:28.189900692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:54:28.747809139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:54:28.753646012Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:54:29.963765829Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T04:54:30.505333806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:54:30.511163661Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:54:33.190104845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:34.318818472Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-21T04:54:34.533927586Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:54:37.195322912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T04:54:38.189643929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:38.673853856Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:54:43.02820468Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:54:43.189985029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:47.383922217Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:54:48.189969864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:54:49.020138168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:54:51.73866977Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=info ts=2026-04-21T04:54:53.173823246Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:54:53.174564301Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T04:54:53.189772791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:54:53.189775321Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:54:53.189792214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:54:53.189799414Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:54:53.617086802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:54:53.625358081Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:54:56.093412794Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=warn ts=2026-04-21T04:54:58.18941511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:54:58.335786548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:54:58.340348536Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:55:00.447558703Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T04:55:03.189760636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:04.802776218Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-21T04:55:05.786739117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:55:05.796828004Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:55:07.197449804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:55:08.189404729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:09.158173262Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=debug ts=2026-04-21T04:55:11.303583601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:55:13.190032049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:13.512247772Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T04:55:14.290498457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52522"
level=debug ts=2026-04-21T04:55:17.867451811Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T04:55:18.189721079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:55:21.85920278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:55:21.864531831Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:55:21.864554122Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:55:21.864582741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:55:21.86818947Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:55:22.222769012Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=warn ts=2026-04-21T04:55:23.189197807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:55:23.189247502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:55:23.189270651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:55:23.189280219Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:55:24.000476889Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:55:25.100766855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37014"
level=debug ts=2026-04-21T04:55:26.577304143Z caller=checkpoint.go:378 msg="writing series" size="702 kB"
level=debug ts=2026-04-21T04:55:26.577326016Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=18
level=warn ts=2026-04-21T04:55:28.189630122Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:30.944837241Z caller=checkpoint.go:378 msg="writing series" size="12 MB"
level=debug ts=2026-04-21T04:55:30.944865114Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-04-21T04:55:33.190178457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:35.286273262Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=info ts=2026-04-21T04:55:36.880760609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:55:36.891489448Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T04:55:37.20041856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:55:38.189531812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:39.641369015Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=warn ts=2026-04-21T04:55:43.189889575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:43.9960928Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=warn ts=2026-04-21T04:55:48.190165959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:55:48.351621383Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=info ts=2026-04-21T04:55:49.219804889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:55:49.226576594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:55:52.705985715Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T04:55:52.828877459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:55:52.834367869Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:55:53.172601781Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:55:53.173672326Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T04:55:53.190017433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:55:53.190023729Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:55:53.190045796Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:55:53.190055364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:55:54.550934078Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T04:55:55.104236093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56054"
level=debug ts=2026-04-21T04:55:56.432727111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:55:57.060789903Z caller=checkpoint.go:378 msg="writing series" size="8.9 kB"
level=warn ts=2026-04-21T04:55:58.19018395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:56:01.415900124Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:56:03.190122213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:56:05.278261638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:56:05.287521091Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:56:05.287545777Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:56:05.287575009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:56:05.293921178Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:56:05.770553716Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
ts=2026-04-21T04:56:07.203290164Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:56:08.189813884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:56:10.125566725Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T04:56:10.661276081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:56:10.667402009Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:56:13.189323046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:56:14.479968208Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
ts=2026-04-21T04:56:15.806535235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:45882"
level=warn ts=2026-04-21T04:56:18.190029336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:56:18.835087998Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=info ts=2026-04-21T04:56:19.290580988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:56:19.297878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T04:56:23.189399242Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:56:23.189436389Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-21T04:56:23.189441909Z caller=checkpoint.go:390 msg="flushing series" totalSize="936 kB" series=11
level=info ts=2026-04-21T04:56:23.189455401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:56:23.189466563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:56:23.195560823Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-21T04:56:23.197284692Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007919163s
ts=2026-04-21T04:56:25.107752961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52736"
level=warn ts=2026-04-21T04:56:28.18926793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:56:29.047771305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:56:30.368670124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:56:33.189254731Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:56:35.0023368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:56:35.006534239Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T04:56:37.20679536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:56:38.189406617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:56:41.881278671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:56:41.889423958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T04:56:43.189694216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:56:44.29812763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50126"
ts=2026-04-21T04:56:45.808333905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:58756"
level=info ts=2026-04-21T04:56:47.352325376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:56:47.358240925Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:56:48.190138995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:56:53.172953785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:56:53.174030457Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:56:53.189257169Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T04:56:53.189300819Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:56:53.189315162Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:56:53.189315338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:56:53.189335103Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T04:56:53.189407429Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-04-21T04:56:53.189751059Z caller=checkpoint.go:378 msg="writing series" size="393 kB"
level=debug ts=2026-04-21T04:56:57.544674162Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T04:56:58.18965786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:01.900265143Z caller=checkpoint.go:378 msg="writing series" size="666 kB"
level=debug ts=2026-04-21T04:57:01.90028847Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T04:57:03.18955849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:03.915679666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T04:57:06.254167105Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T04:57:07.209700457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:57:08.189351683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:57:10.384259989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:57:10.389711374Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:57:10.609365443Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T04:57:13.190101872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:14.964166103Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
ts=2026-04-21T04:57:15.80996424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:55340"
level=warn ts=2026-04-21T04:57:18.189373966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:19.319028253Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-04-21T04:57:19.731096986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:57:19.74010072Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:57:22.450450562Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T04:57:22.452074762Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T04:57:23.189663107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:57:23.189688151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:57:23.189696315Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:57:23.189686222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:23.673727456Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=info ts=2026-04-21T04:57:24.228625039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:57:24.235726724Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:57:25.113246023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:44700"
level=debug ts=2026-04-21T04:57:28.02850977Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T04:57:28.189333229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:32.383238568Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=debug ts=2026-04-21T04:57:32.798653587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T04:57:33.190105031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:36.740826919Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-21T04:57:37.212113359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T04:57:38.190113416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:41.093062285Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=warn ts=2026-04-21T04:57:43.190066327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:57:44.303077566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35002"
level=info ts=2026-04-21T04:57:44.783694147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:57:44.790944107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:57:45.448017118Z caller=checkpoint.go:378 msg="writing series" size="215 kB"
level=warn ts=2026-04-21T04:57:48.189424456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:49.802838171Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=info ts=2026-04-21T04:57:53.173073983Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:57:53.173442752Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:57:53.189726306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:57:53.189737361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:53.18976002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:57:53.189769477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:57:54.158340132Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-21T04:57:58.189532846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:57:58.512832833Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=debug ts=2026-04-21T04:57:58.512857294Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=13
level=debug ts=2026-04-21T04:58:02.867259708Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T04:58:03.190194551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:58:07.214816655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T04:58:07.222274519Z caller=checkpoint.go:378 msg="writing series" size="810 B"
level=debug ts=2026-04-21T04:58:07.908335282Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:58:08.18152281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:58:08.185192802Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:58:08.189188316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:10.228679062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:58:11.576742745Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T04:58:13.189756245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:58:13.482795456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:58:13.486677095Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T04:58:14.305427089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45402"
level=info ts=2026-04-21T04:58:14.518349536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:58:14.525007288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:58:15.93197766Z caller=checkpoint.go:378 msg="writing series" size="8.8 kB"
level=warn ts=2026-04-21T04:58:18.189406968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:20.286610356Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-21T04:58:23.190051106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:58:23.19009426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T04:58:23.190090275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:58:23.190120112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:58:24.641620462Z caller=checkpoint.go:378 msg="writing series" size="322 kB"
level=warn ts=2026-04-21T04:58:28.190040464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:28.997689187Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T04:58:28.997711892Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=7
level=info ts=2026-04-21T04:58:31.911236823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:58:31.919276211Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T04:58:33.189613025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:33.351005028Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=debug ts=2026-04-21T04:58:34.318190797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:58:37.217329693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T04:58:37.705974571Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T04:58:38.190232136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:42.060651042Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T04:58:43.190100298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:58:44.308468167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:43978"
level=debug ts=2026-04-21T04:58:46.415788033Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=warn ts=2026-04-21T04:58:48.190153856Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:50.770606381Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=info ts=2026-04-21T04:58:53.173103867Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T04:58:53.174172281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:58:53.189411137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T04:58:53.189408792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:53.189431432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:58:53.189439214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T04:58:55.126169829Z caller=checkpoint.go:378 msg="writing series" size="276 kB"
level=warn ts=2026-04-21T04:58:58.18943419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:58:59.48087421Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-04-21T04:58:59.480900942Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=7
level=info ts=2026-04-21T04:59:01.083910086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:59:01.089749474Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T04:59:03.190043059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:59:03.83548728Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=info ts=2026-04-21T04:59:07.018798039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:59:07.02648856Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T04:59:07.220043612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T04:59:07.460009801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T04:59:08.189806536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:59:08.190089836Z caller=checkpoint.go:378 msg="writing series" size="516 kB"
level=debug ts=2026-04-21T04:59:08.580016948Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T04:59:12.475770239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:59:12.480123167Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:59:12.480147618Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T04:59:12.480170696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:59:12.483269149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:59:12.545559568Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-04-21T04:59:13.189199558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:59:14.311470823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54730"
level=debug ts=2026-04-21T04:59:16.899929313Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-21T04:59:18.19002177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:59:21.254188635Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-21T04:59:23.189317157Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:59:23.189342897Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:59:23.189350905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:59:23.189356974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T04:59:25.119822946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48144"
level=debug ts=2026-04-21T04:59:25.609908996Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T04:59:27.478410228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:59:27.485920611Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T04:59:28.190171011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:59:28.470272239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T04:59:28.475926469Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T04:59:29.964048379Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T04:59:33.189902039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:59:34.319345994Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-21T04:59:37.222639989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T04:59:38.189359314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:59:38.673828841Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T04:59:43.028832253Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-21T04:59:43.18975982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T04:59:43.337243712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T04:59:44.314128042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52542"
level=debug ts=2026-04-21T04:59:47.271286603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T04:59:47.383699746Z caller=checkpoint.go:378 msg="writing series" size="4.2 kB"
level=warn ts=2026-04-21T04:59:48.190078472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:59:51.49092564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T04:59:51.497417036Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T04:59:51.738476277Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T04:59:52.035345102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T04:59:52.039507649Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T04:59:53.172655295Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T04:59:53.173701071Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T04:59:53.190011765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T04:59:53.190035448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T04:59:53.190044008Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T04:59:53.190047837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T04:59:54.232516967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T04:59:54.236737758Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T04:59:56.093762248Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=debug ts=2026-04-21T04:59:56.093789775Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-21T04:59:58.190076287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:00.447707423Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T05:00:03.189579515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:04.802654516Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-21T05:00:07.224427306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:00:08.189248402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:09.158147555Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=info ts=2026-04-21T05:00:12.204897234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:00:12.211706524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:00:13.189894147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:13.512941546Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-21T05:00:14.322365128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54662"
ts=2026-04-21T05:00:15.826608655Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:34452"
level=debug ts=2026-04-21T05:00:16.586772443Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:00:17.867763826Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-21T05:00:18.189868873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:22.222962111Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=info ts=2026-04-21T05:00:23.18975857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:00:23.189755528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:23.189784696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:00:23.189793567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:00:26.577099863Z caller=checkpoint.go:378 msg="writing series" size="805 kB"
level=debug ts=2026-04-21T05:00:26.577124065Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-21T05:00:28.18993319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:30.942808226Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-21T05:00:30.94283184Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=1
level=warn ts=2026-04-21T05:00:33.190026528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:00:33.34134853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:00:33.345290361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:00:34.126614004Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:00:35.286873298Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=info ts=2026-04-21T05:00:35.674936583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:00:35.678881553Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T05:00:37.226957255Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:00:38.189659568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:39.641706355Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-21T05:00:43.190106495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:00:43.267400542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:00:43.276342631Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:00:43.996636419Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=warn ts=2026-04-21T05:00:48.189992555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:00:48.351506805Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=debug ts=2026-04-21T05:00:50.278050163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:00:52.706004775Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T05:00:53.173058419Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:00:53.173980152Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=debug ts=2026-04-21T05:00:53.17417686Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T05:00:53.174240434Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974163 n=1
level=debug ts=2026-04-21T05:00:53.174283407Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974163 n=1
level=warn ts=2026-04-21T05:00:53.189261795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:00:53.189274706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:00:53.189299912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:00:53.189311648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:00:57.060653634Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-21T05:00:58.0762897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:00:58.083767876Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:00:58.083787469Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:00:58.0838097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:00:58.089124317Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:00:58.189887669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:01:01.415708744Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-21T05:01:03.18960711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:01:05.770645616Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
ts=2026-04-21T05:01:07.230374824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:01:08.189442299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:01:10.125340341Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T05:01:10.62696434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:01:10.635589718Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:01:13.189456449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:01:13.498751607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:01:13.503344349Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:01:14.480202387Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-21T05:01:18.190129522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:01:18.835797264Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=debug ts=2026-04-21T05:01:19.117129444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:01:19.119288394Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T05:01:21.521264602Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:01:23.189187005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:01:23.189238201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:01:23.189246937Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:01:23.189257509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:01:23.190170267Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=11
level=info ts=2026-04-21T05:01:23.197966304Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-21T05:01:23.199927482Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010392588s
level=warn ts=2026-04-21T05:01:28.189622163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:01:29.155259857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:01:29.161182543Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:01:29.601277091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:01:29.608659218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T05:01:33.189265769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:01:37.233261628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:01:38.189263884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T05:01:43.189248366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:01:43.21033743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:01:43.218529992Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T05:01:44.329140345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44590"
level=warn ts=2026-04-21T05:01:48.190101018Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:01:51.613052946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:01:53.172842522Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:01:53.173903967Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:01:53.189124521Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T05:01:53.189172891Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:01:53.189240413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:01:53.189253864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:01:53.189265843Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:01:53.189348153Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=debug ts=2026-04-21T05:01:53.189799109Z caller=checkpoint.go:378 msg="writing series" size="418 kB"
level=debug ts=2026-04-21T05:01:57.545416455Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-21T05:01:58.189836041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:01.900547381Z caller=checkpoint.go:378 msg="writing series" size="707 kB"
level=debug ts=2026-04-21T05:02:01.900570421Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T05:02:03.189200643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:02:03.851170457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:02:03.85798208Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:02:06.254684602Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T05:02:07.23577886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:02:08.189420631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:10.609272383Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T05:02:13.189788894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:14.964578181Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=debug ts=2026-04-21T05:02:17.489928051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:02:18.19024401Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:02:18.972147024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:02:18.980378943Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:02:18.98040704Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:02:18.980439294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:02:18.986967682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:02:19.31902072Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=debug ts=2026-04-21T05:02:21.566782193Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:02:23.189402669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:02:23.189410175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:02:23.189432884Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:02:23.18944384Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:02:23.674244955Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:02:26.251167524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:02:26.256497846Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:02:28.029354036Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-21T05:02:28.189838602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:02:28.490028477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:02:28.494015363Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:02:28.494036889Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:02:28.494061384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:02:28.4974823Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:02:32.38399815Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T05:02:33.189295142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:36.738650669Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-04-21T05:02:37.119160307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:02:37.126030853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T05:02:37.237653844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:02:38.189640489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:41.093680106Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-04-21T05:02:43.190154406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:45.448624338Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-04-21T05:02:47.611349589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:02:47.611400054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-21T05:02:47.615737163Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:02:47.911546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="201 B" avg_comp="201 B" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:02:47.915712581Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:02:48.190038333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:02:48.212224002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="155 B" avg_comp="155 B" total_uncomp="15 kB" avg_uncomp="15 kB" idle=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:02:48.215776564Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=info ts=2026-04-21T05:02:48.261850811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:02:48.267440741Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:02:48.512145232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:02:48.515589571Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:02:48.812423432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:02:48.816293025Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:02:49.112044006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:02:49.115857107Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:02:49.411880569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:02:49.415346297Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:02:49.712202559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:02:49.715653657Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:02:49.803582731Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=debug ts=2026-04-21T05:02:49.803606094Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T05:02:50.011659816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:02:50.01537313Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:02:50.312415483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="55 kB" avg_uncomp="55 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:02:50.316073385Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:02:50.612066554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:02:50.616756471Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:02:50.911407329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:02:50.91534038Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:02:51.211573435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:02:51.21527149Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:02:51.790108045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:02:51.794404941Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:02:53.173006361Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:02:53.173464301Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:02:53.189756655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:02:53.189756499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:53.189777316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:02:53.189785212Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:02:54.158326463Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
ts=2026-04-21T05:02:55.138668289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:60650"
level=warn ts=2026-04-21T05:02:58.189996069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:02:58.513672186Z caller=checkpoint.go:378 msg="writing series" size="467 kB"
level=info ts=2026-04-21T05:02:59.895834029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:02:59.901960802Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=info ts=2026-04-21T05:03:00.509760286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:03:00.513108442Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:03:01.550200602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:03:01.553659705Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:03:02.104260344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:03:02.107613057Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:03:02.868041443Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T05:03:02.895164822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:03:02.898484474Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:03:03.189613639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:03.210771691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:03:03.214117284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:03:03.96269759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:03:03.9671574Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:03:04.356019694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:03:04.360692268Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:03:05.63648649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:03:05.639889023Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:03:07.057459173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:03:07.060840058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:03:07.222541241Z caller=checkpoint.go:378 msg="writing series" size="810 B"
ts=2026-04-21T05:03:07.240807039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:03:07.358349136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:03:07.361789381Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:03:07.373028443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:03:07.375972727Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:03:07.395040526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:03:07.398092524Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:03:08.189661918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:09.869013969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:03:09.874834886Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:03:10.257443813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:03:10.264681849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:03:11.577112745Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T05:03:13.189749704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:13.798500028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:03:13.802871035Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:03:14.651362664Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:03:14.651431354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:03:14.654915523Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:03:15.932363632Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-21T05:03:18.190009811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:03:20.286794003Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-21T05:03:21.626610643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:03:23.189345884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:03:23.189345956Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:03:23.189372767Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:03:23.189380773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:03:24.64227962Z caller=checkpoint.go:378 msg="writing series" size="352 kB"
level=info ts=2026-04-21T05:03:25.175835867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:03:25.180511677Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:03:25.381102639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:03:25.384809538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:03:28.189580272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:03:28.998645736Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-21T05:03:28.998699076Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=9
level=info ts=2026-04-21T05:03:29.232415209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:03:29.235798503Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:03:30.138962456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:03:30.143057261Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:03:30.222544191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:03:30.226377147Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:03:31.169197552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:03:31.173054322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-04-21T05:03:33.190050071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:03:33.351534626Z caller=checkpoint.go:378 msg="writing series" size="290 kB"
level=info ts=2026-04-21T05:03:34.43242953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:03:34.437464295Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-21T05:03:37.24311833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:03:37.561589846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:03:37.565725596Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:03:37.701072682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:03:37.704944972Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:03:37.705948425Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:03:37.79242133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:03:37.795728726Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:03:38.189681646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:38.228859913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:03:38.232141585Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=debug ts=2026-04-21T05:03:39.03927481Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:03:39.039351693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="59 kB" avg_comp="59 kB" total_uncomp="574 kB" avg_uncomp="574 kB" idle=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:03:39.043061587Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:03:40.178777298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:03:40.182247434Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:03:40.512077398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:03:40.515387312Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:03:42.060830653Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T05:03:43.189750136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:45.940669694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:03:45.944002263Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:03:46.415742255Z caller=checkpoint.go:378 msg="writing series" size="160 kB"
level=warn ts=2026-04-21T05:03:48.189666718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:49.551938404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:03:49.556893743Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:03:50.424688781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:03:50.430402762Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:03:50.770720994Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=info ts=2026-04-21T05:03:51.418355332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:03:51.421960966Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:03:52.27615773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:03:52.28375002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:03:53.173096895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:03:53.17349822Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:03:53.189808159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:03:53.189814255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:03:53.189840236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:03:53.189851124Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:03:55.126040518Z caller=checkpoint.go:378 msg="writing series" size="366 kB"
ts=2026-04-21T05:03:55.144393946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:57358"
level=warn ts=2026-04-21T05:03:58.189629461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:03:58.80168936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:03:58.805788531Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:03:59.480411671Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-21T05:04:01.252608698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:04:01.254444219Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:04:03.186135839Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:04:03.189329776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:04:03.835497693Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
ts=2026-04-21T05:04:07.244962488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:04:08.189316964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:04:08.190583752Z caller=checkpoint.go:378 msg="writing series" size="381 kB"
level=debug ts=2026-04-21T05:04:08.190600477Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=9
level=debug ts=2026-04-21T05:04:08.51237063Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:04:08.512431761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="142 kB" avg_uncomp="142 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:04:08.51807012Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:04:12.545189501Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-21T05:04:13.189246927Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:04:14.671258838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:04:14.675443251Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:04:15.229326597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:04:15.233681393Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:04:15.238730353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:04:15.24197031Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:04:15.75540186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:04:15.758803386Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:04:16.900273245Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=warn ts=2026-04-21T05:04:18.189963257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:04:18.56107745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:04:18.564886874Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:04:19.262352565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:04:19.265754749Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:04:21.254351614Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:04:23.189956296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:04:23.18998363Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:04:23.189999306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:04:23.190029405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:04:25.139998554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:04:25.143701102Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-04-21T05:04:25.146687744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:54098"
level=debug ts=2026-04-21T05:04:25.60956742Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:04:27.177900389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:04:27.181762471Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:04:27.301025326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:04:27.30455847Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:04:27.332666834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:04:27.336329598Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:04:28.189393063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:04:29.964777266Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T05:04:30.771462558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:04:30.775038356Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:04:32.056449224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:04:32.060346108Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:04:33.045250974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:04:33.050264889Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:04:33.189983071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:04:34.319362086Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T05:04:36.267766046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:04:36.271590799Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:04:37.099431314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T05:04:37.246130721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:04:37.520420664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:04:37.523710905Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=warn ts=2026-04-21T05:04:38.190097921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:04:38.673728166Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T05:04:43.029317391Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-21T05:04:43.190060272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:04:44.343763351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52976"
level=info ts=2026-04-21T05:04:44.670351923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:04:44.67510197Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:04:45.02605264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:04:45.029817819Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:04:47.383440458Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-21T05:04:48.189329713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:04:48.945659632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:04:48.952186797Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:04:51.738118235Z caller=checkpoint.go:378 msg="writing series" size="120 kB"
level=info ts=2026-04-21T05:04:53.173036811Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:04:53.173565931Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T05:04:53.189867474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:04:53.189943625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:04:53.189970367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:04:53.189981445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:04:53.284339289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:04:53.289344212Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:04:53.831336877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:04:53.835293615Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:04:56.093133661Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-04-21T05:04:56.214742006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:04:56.218251267Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:04:57.950990362Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:04:58.189668193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:04:58.627715761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:04:58.631374873Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:04:59.6010619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:04:59.605142777Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:04:59.902877131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:04:59.906382286Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:05:00.448115216Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=debug ts=2026-04-21T05:05:01.781507797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:05:02.750037293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:05:02.754509362Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:05:03.189414705Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:05:03.600418802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:05:03.603891086Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:05:04.803287797Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
ts=2026-04-21T05:05:07.247938364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:05:08.189641294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:05:08.467154003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:05:08.470934581Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:05:09.157940166Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=info ts=2026-04-21T05:05:11.598564272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:05:11.602363171Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:05:11.985456584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:05:11.98919778Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:05:13.190006142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:05:13.513066596Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
ts=2026-04-21T05:05:15.856266836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:48370"
level=debug ts=2026-04-21T05:05:17.86768077Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:05:18.189941747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:05:19.219521418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:05:19.226280333Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:05:22.222894034Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:05:23.189710254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:05:23.189711805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:05:23.189735011Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:05:23.18974373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:05:23.217886799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:05:23.221274513Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:05:23.270410227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:05:23.273809286Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T05:05:25.152511906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49012"
level=debug ts=2026-04-21T05:05:26.577715281Z caller=checkpoint.go:378 msg="writing series" size="925 kB"
level=debug ts=2026-04-21T05:05:26.577737029Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=18
level=warn ts=2026-04-21T05:05:28.190004557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:05:28.654747552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:05:28.66013253Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:05:28.671248362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:05:28.6748404Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:05:28.674856241Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:05:28.674878221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:05:28.678460119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:05:30.678620777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:05:30.682046673Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:05:30.945105937Z caller=checkpoint.go:378 msg="writing series" size="14 MB"
level=debug ts=2026-04-21T05:05:30.945132306Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=1
level=debug ts=2026-04-21T05:05:31.94502626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:05:33.189271896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:05:34.754326235Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:05:34.755999454Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T05:05:35.286696806Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=info ts=2026-04-21T05:05:35.5906334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:05:35.594322892Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
ts=2026-04-21T05:05:37.251459927Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:05:38.19014547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:05:38.6333494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:05:38.636912002Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:05:39.448714232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:05:39.452285353Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:05:39.641812893Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=warn ts=2026-04-21T05:05:43.189404681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:05:43.997077572Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=info ts=2026-04-21T05:05:44.287860839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:05:44.291527811Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-21T05:05:44.348115167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:43624"
ts=2026-04-21T05:05:45.857915909Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:51074"
level=info ts=2026-04-21T05:05:46.874099725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:05:46.879411855Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:05:46.896481226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:05:46.900227057Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:05:48.004427847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:05:48.008571889Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:05:48.189340046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:05:48.351834448Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=info ts=2026-04-21T05:05:48.512886031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:05:48.51973593Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:05:49.991652898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:05:49.99704922Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:05:52.129908655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:05:52.134436921Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:05:52.706273069Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-04-21T05:05:53.172804932Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:05:53.173848356Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:05:53.191279353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:05:53.19128312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:05:53.191303936Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:05:53.191312029Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:05:53.834974083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:05:53.838468859Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:05:57.061394084Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T05:05:58.189369957Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:05:58.377396769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:05:58.381308907Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:05:59.168481768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:05:59.172071157Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:06:00.388424111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:06:01.41582457Z caller=checkpoint.go:378 msg="writing series" size="6.2 kB"
level=warn ts=2026-04-21T05:06:03.189888865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:06:05.771198883Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=info ts=2026-04-21T05:06:05.888357163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:06:05.893783543Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-04-21T05:06:07.254972642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:06:07.594883337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:06:07.598841296Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:06:08.190190896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:09.514572083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:06:09.520132101Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:06:10.12597822Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T05:06:12.507790938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:06:12.511889249Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-04-21T05:06:13.18981982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:06:14.480956339Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-21T05:06:15.371254899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:06:15.85932893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:35422"
level=info ts=2026-04-21T05:06:15.963060335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:06:15.966392325Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:06:16.323788117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:06:16.327335358Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:06:18.189999966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:06:18.835377589Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=debug ts=2026-04-21T05:06:18.835403898Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T05:06:20.436754447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:06:20.442114519Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:06:20.523358007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:06:20.526807805Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:06:23.190136767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:06:23.190165365Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:06:23.190174165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:06:23.19019451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:23.195386662Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-04-21T05:06:23.197457214Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007913071s
level=info ts=2026-04-21T05:06:26.217012854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:06:26.221013508Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:06:28.189292143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:29.530390815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:06:29.534376662Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:06:30.09305426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:06:30.096650449Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=info ts=2026-04-21T05:06:31.193803702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:06:31.197817129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T05:06:33.189805163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:06:34.00194757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T05:06:37.258407825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:06:37.846544679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:06:37.850723476Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:06:38.189505653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:41.380784587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:06:41.384467298Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:06:43.190152383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:43.85581438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:06:43.863378233Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:06:44.083575682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:06:44.08731895Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-04-21T05:06:45.860662333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42140"
level=warn ts=2026-04-21T05:06:48.189398616Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:49.099747729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:06:49.10365627Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:06:49.944705511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:06:49.948652297Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:06:53.172956773Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:06:53.173995034Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:06:53.189274132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T05:06:53.189273991Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-21T05:06:53.189304489Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:06:53.189324623Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:06:53.189330629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:06:53.189450316Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=debug ts=2026-04-21T05:06:53.189866715Z caller=checkpoint.go:378 msg="writing series" size="429 kB"
level=info ts=2026-04-21T05:06:53.678935626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:06:53.685321504Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:06:53.880568424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:06:53.88413809Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:06:57.545164916Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-04-21T05:06:57.62360478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:06:57.627514017Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:06:58.189801437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:01.61925286Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:07:01.900435334Z caller=checkpoint.go:378 msg="writing series" size="832 kB"
level=debug ts=2026-04-21T05:07:01.900458607Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-04-21T05:07:02.143660502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:07:02.147268625Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:07:03.189849019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:07:04.277029253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:07:04.282119064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:07:04.282144933Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:07:04.282171313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:07:04.287476689Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:07:04.380913418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:07:04.384184571Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:07:04.940363767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:07:05.055804341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:07:05.059185946Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:07:06.254676953Z caller=checkpoint.go:378 msg="writing series" size="654 B"
level=info ts=2026-04-21T05:07:06.688689585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:07:06.692547767Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:07:07.152762354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:07:07.15758132Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-04-21T05:07:07.261369739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:07:08.189625254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:10.609733121Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-21T05:07:13.18927637Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:07:14.555234457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:07:14.558489515Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:07:14.963909304Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T05:07:15.322989625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:07:15.32679656Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:07:16.975764176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:07:16.979944726Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T05:07:18.189253299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:19.319313797Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=info ts=2026-04-21T05:07:20.186882093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:07:20.193755779Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:07:22.119749072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:07:22.123353367Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:07:23.190069822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:07:23.190099725Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:07:23.19010849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:07:23.190107281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:07:23.480395153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:07:23.484040341Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:07:23.674516713Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:07:26.870881738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:07:26.875793191Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=info ts=2026-04-21T05:07:27.129331469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:07:27.132624666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:07:28.029253694Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-04-21T05:07:28.190017544Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:32.383971787Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T05:07:33.189906593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:35.257795456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:07:36.738266143Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
ts=2026-04-21T05:07:37.263940068Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:07:38.189562423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:07:38.914920868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:07:38.920583821Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:07:39.511586899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:07:39.515648643Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:07:40.129645747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:07:40.133476932Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:07:40.453763883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:07:40.460281833Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:07:41.093636332Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=info ts=2026-04-21T05:07:41.395240432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:07:41.398706912Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:07:42.74144327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:07:43.189319105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:45.447892365Z caller=checkpoint.go:378 msg="writing series" size="229 kB"
level=info ts=2026-04-21T05:07:46.839798119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:07:46.843628333Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:07:47.145348069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:07:47.148669449Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:07:47.431727012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:07:47.435332409Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:07:47.511569832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:07:47.516682447Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:07:47.803580916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:07:47.80694273Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:07:48.189418094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:49.803324204Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=debug ts=2026-04-21T05:07:49.803347851Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T05:07:52.146141291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:07:52.15003652Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:07:53.173019353Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:07:53.173428929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=warn ts=2026-04-21T05:07:53.189730155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:07:53.189728655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:07:53.18975465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:07:53.189763444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:07:54.15751632Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-04-21T05:07:54.932694807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:07:54.936517829Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:07:58.19014221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:07:58.513386404Z caller=checkpoint.go:378 msg="writing series" size="385 kB"
level=info ts=2026-04-21T05:08:02.185406654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:08:02.190280372Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:08:02.867563797Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T05:08:03.01304024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:08:03.017589535Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-04-21T05:08:03.189686436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:08:03.82563398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:08:03.82948751Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:08:07.222916152Z caller=checkpoint.go:378 msg="writing series" size="821 B"
ts=2026-04-21T05:08:07.266338514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:08:08.189273156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:08:08.787335478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:08:08.790775937Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:08:10.619006352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:08:10.623103878Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:08:10.887997699Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:08:10.888061274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="252 B" avg_comp="252 B" total_uncomp="107 kB" avg_uncomp="107 kB" idle=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:08:10.891653591Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:08:11.335478538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:08:11.339498406Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:08:11.57755924Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T05:08:13.189493736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:08:13.459698476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:08:13.463361881Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T05:08:15.868241818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:41266"
level=debug ts=2026-04-21T05:08:15.931988237Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-21T05:08:18.190076623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:20.286567446Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T05:08:23.19013926Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:08:23.190165465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:08:23.190173696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:08:23.190188835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:24.642307002Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=info ts=2026-04-21T05:08:25.058332673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:08:25.065843148Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=info ts=2026-04-21T05:08:27.040025673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:08:27.044549203Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:08:27.515027976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:08:27.519475735Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-04-21T05:08:28.189673931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:28.997837084Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T05:08:28.997860439Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=9
level=info ts=2026-04-21T05:08:29.527552515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:08:29.531726164Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:08:30.821752638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:08:30.825558742Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:08:31.864883778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:08:31.868559417Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:08:33.039666852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:08:33.047179079Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:08:33.10152207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:08:33.104754083Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:08:33.18928454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:33.351754765Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-21T05:08:34.271084759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="405 B" avg_comp="405 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:08:34.274595796Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:08:35.346391311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:08:35.353675938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:08:36.887319855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:08:36.891463099Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-04-21T05:08:37.268960437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:08:37.7065133Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T05:08:38.190225347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:38.432326651Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:08:38.893237311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:08:38.896666569Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:08:41.352298556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:08:42.060698866Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T05:08:43.189285298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:08:43.535817498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:08:43.539327802Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T05:08:44.368126862Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37828"
level=info ts=2026-04-21T05:08:44.406707351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:08:44.410331328Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:08:46.019051696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:08:46.023184993Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:08:46.416652846Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=warn ts=2026-04-21T05:08:48.189979663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:50.77109989Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=debug ts=2026-04-21T05:08:53.172649463Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:08:53.173697612Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:08:53.190023845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:08:53.190056077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:08:53.190066668Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:08:53.190077515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:08:55.12540308Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
ts=2026-04-21T05:08:55.170118535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:48730"
level=info ts=2026-04-21T05:08:55.806421357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:08:55.811344011Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:08:58.189881892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:08:59.380274993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:08:59.383997552Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:08:59.480362855Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=info ts=2026-04-21T05:09:02.536713421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:09:02.541349968Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:09:02.997663992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:09:03.00118282Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T05:09:03.190026382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:09:03.282417922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:09:03.835878612Z caller=checkpoint.go:378 msg="writing series" size="4.5 kB"
level=info ts=2026-04-21T05:09:05.1911691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:09:05.19604022Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=info ts=2026-04-21T05:09:06.472080372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:09:06.475462241Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:09:06.475485694Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:06.475510883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:09:06.478674759Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:09:06.552558475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:09:06.555740803Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:09:06.555760499Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:06.77594098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:09:06.779431584Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T05:09:07.271491725Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:09:08.19022481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:09:08.190428537Z caller=checkpoint.go:378 msg="writing series" size="454 kB"
level=debug ts=2026-04-21T05:09:08.190445236Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-04-21T05:09:08.570117463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:09:08.573955068Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:09:08.573976565Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:08.57400061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:09:08.577361097Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:09:10.143937231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:09:10.147333287Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-04-21T05:09:10.147352581Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:10.147374768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:09:10.150234937Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:09:12.545669785Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=warn ts=2026-04-21T05:09:13.189831886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:09:14.371457377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50660"
ts=2026-04-21T05:09:15.872500189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:50110"
level=info ts=2026-04-21T05:09:16.24733043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:09:16.25076022Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:09:16.900014291Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-21T05:09:17.128311885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:09:17.12993631Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T05:09:18.189810746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:18.651037577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:09:18.65607937Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:09:19.114077029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:09:19.118245362Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T05:09:19.735164109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:09:19.738669227Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:09:21.255064129Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:09:22.815862134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:09:22.819532091Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:09:23.189950101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:09:23.18997841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:09:23.18998774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:09:23.18997962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:23.818417137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:09:23.821818849Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:09:23.927096714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:09:23.930619998Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-04-21T05:09:23.930640126Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:23.930662158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:09:23.934194174Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:09:24.241495931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:09:24.24818966Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:09:24.24823864Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:24.2482782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:09:24.254345559Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:09:25.077759274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:09:25.081127731Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:09:25.609772175Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:09:26.727351558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:09:26.732606543Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T05:09:26.732631403Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:26.732654635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:09:26.735544642Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:09:27.270401729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:09:27.273634877Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:09:27.273657693Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:27.273681309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:09:27.276366447Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:09:27.546026391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:09:27.549276973Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:09:27.549297845Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:27.574650439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:09:27.577907979Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:09:28.19015881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:28.534918923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:09:28.538280355Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:09:28.538304389Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:28.538331538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:09:28.541234012Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:09:29.964618784Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T05:09:33.190197218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:33.909900333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:09:33.914100401Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:09:34.319271121Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-21T05:09:36.635153171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T05:09:37.274005059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:09:38.190108499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:09:38.674286877Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T05:09:40.639974201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:09:40.6502886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:09:41.221631423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:09:41.225143672Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:09:41.789318509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:09:41.792840307Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:09:41.792860061Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:41.792881855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:09:41.795574254Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:09:43.029155059Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-21T05:09:43.190063788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:43.761196848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:09:43.764917868Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-04-21T05:09:43.764940987Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:43.7649653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:09:43.767850631Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:09:44.292394753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:09:44.296248068Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-04-21T05:09:44.374316623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60134"
level=info ts=2026-04-21T05:09:45.192580873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:09:45.196335846Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:09:45.196360095Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:45.196383963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:09:45.199541722Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:09:45.319827916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:09:45.323682886Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:09:45.626805441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:09:45.630468793Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:09:47.38401923Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-04-21T05:09:48.153005786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:09:48.156936497Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:09:48.190012694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:50.851326709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:09:50.85502796Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:09:51.465260289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:09:51.468835788Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:09:51.73886088Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-04-21T05:09:52.892109488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:09:52.895649376Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:09:52.895670839Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:52.895693678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:09:52.898546282Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:09:52.93562232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:09:52.939388913Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:09:53.172508537Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:09:53.173571026Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T05:09:53.189880506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:53.189892186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:09:53.18991255Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:09:53.189920802Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:09:54.37922743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:09:54.382624309Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:09:54.512808755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=794568c2718d6d21 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:09:54.515978621Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/794568c2718d6d21/19dae340f1b%3A19dae340f1c%3A3b3c822d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-kueue\"}"
level=error ts=2026-04-21T05:09:54.515997705Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:09:54.516017585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:09:54.518696007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:09:55.264941569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:09:55.268285899Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:09:55.37167744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:09:55.375121464Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:09:56.093357877Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=info ts=2026-04-21T05:09:56.427819598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:09:56.431094991Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-04-21T05:09:58.190176856Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:09:58.575906741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:09:58.579962991Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:09:59.591226427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:09:59.594830678Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:10:00.448071465Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T05:10:03.189238442Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:04.803257431Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T05:10:06.992843391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:10:06.996355415Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-04-21T05:10:07.275473056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T05:10:07.509745541Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:10:08.189885793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:10:08.273176471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:10:08.276985647Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:10:09.15826314Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=info ts=2026-04-21T05:10:09.194649234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:10:09.198157306Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:10:11.281919692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:10:11.290559406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:10:11.364266303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:10:11.368137716Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:10:12.126623212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:10:12.130009961Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:10:13.189791841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:13.513083524Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=debug ts=2026-04-21T05:10:13.627035589Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:10:14.07558193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:10:14.078819816Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:10:14.082880907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:10:14.085957804Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:10:14.824485277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:10:14.829849089Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:10:15.458687523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:10:15.461936428Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:10:15.461960142Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:10:15.461982316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:10:15.464566699Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:10:17.86765308Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:10:18.19023518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:10:20.170471716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:10:20.174313064Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:10:20.174334262Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:10:20.174359744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:10:20.177360107Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:10:22.222702588Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:10:23.189923832Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:10:23.189952652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:10:23.189960798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:10:23.189985061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:10:23.493505023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:10:23.497356077Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:10:23.511443464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:10:23.51151564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:10:23.515018394Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:10:23.515016047Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-04-21T05:10:25.179268262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:59756"
level=debug ts=2026-04-21T05:10:26.57737245Z caller=checkpoint.go:378 msg="writing series" size="793 kB"
level=debug ts=2026-04-21T05:10:26.577395348Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=18
level=info ts=2026-04-21T05:10:27.413478402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:10:27.417369956Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:10:28.19019693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:30.948972233Z caller=checkpoint.go:378 msg="writing series" size="14 MB"
level=debug ts=2026-04-21T05:10:30.949003667Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-04-21T05:10:32.054733891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:10:32.059604309Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:10:33.02335877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:10:33.027285231Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:10:33.190158911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:34.094744536Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:10:35.28678725Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
ts=2026-04-21T05:10:37.276848309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:10:37.701613558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:10:37.705755791Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:10:38.189701783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:10:39.206630107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:10:39.210375171Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:10:39.641587799Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=info ts=2026-04-21T05:10:41.872758194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:10:41.882375877Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:10:42.921262674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:10:42.925187708Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T05:10:43.189942786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:43.996768748Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=info ts=2026-04-21T05:10:44.925238999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:10:44.929288627Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:10:44.929307735Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:10:44.929329177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:10:44.931953586Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T05:10:45.880057579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42480"
level=warn ts=2026-04-21T05:10:48.189782562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:48.35130561Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-04-21T05:10:48.478132965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:10:48.481717343Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:10:50.166921546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:10:50.170640743Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:10:50.466635159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:10:50.470597366Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:10:50.552703433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:10:50.556701979Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:10:52.086729881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:10:52.090232494Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:10:52.706274158Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-04-21T05:10:53.173033514Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:10:53.173442823Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:10:53.189724441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:10:53.189751079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:10:53.18976153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:10:53.189761236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:10:53.83571679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-21T05:10:55.182108984Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35694"
level=info ts=2026-04-21T05:10:55.457102172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:10:55.460832678Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:10:56.795498572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:10:56.798968175Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:10:57.061574439Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T05:10:58.189390147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:00.211156692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="111 MB" avg_uncomp="22 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:11:00.217582227Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:11:01.415741396Z caller=checkpoint.go:378 msg="writing series" size="6.2 kB"
level=info ts=2026-04-21T05:11:02.266352702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:11:02.269670198Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:11:03.189913965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:11:03.583261491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:11:05.771198638Z caller=checkpoint.go:378 msg="writing series" size="311 kB"
ts=2026-04-21T05:11:07.277814221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:11:07.831869764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:11:07.836427656Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:11:08.189633234Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:09.497682459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:11:09.501612835Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:11:09.817492214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:11:09.821389735Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:11:10.079134351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:11:10.084096836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:11:10.125250092Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=debug ts=2026-04-21T05:11:10.125272228Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-21T05:11:13.1892662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:13.807909242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:11:13.812752856Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-04-21T05:11:14.383422859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44192"
level=debug ts=2026-04-21T05:11:14.48095663Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-04-21T05:11:15.881835071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:39948"
level=warn ts=2026-04-21T05:11:18.189431476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:11:18.835249915Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=info ts=2026-04-21T05:11:20.385315547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:11:20.389176737Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:11:21.362711403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:11:21.366381685Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:11:21.92870426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:11:21.93243117Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:11:23.189933824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:11:23.189936335Z caller=checkpoint.go:390 msg="flushing series" totalSize="103 kB" series=2
level=info ts=2026-04-21T05:11:23.189939104Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:11:23.189962865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:11:23.189973217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:11:23.195900513Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-04-21T05:11:23.197805063Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008202628s
ts=2026-04-21T05:11:25.184698437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:46442"
level=info ts=2026-04-21T05:11:26.11676566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:11:26.120751125Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T05:11:28.189687248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:28.430581028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:11:28.434191365Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:11:30.970555222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:11:30.974166244Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:11:31.134568175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:11:31.138786116Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:11:32.595257172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:11:32.600156886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:11:33.190061432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:35.158462714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:11:35.162506693Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-21T05:11:37.280242918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:11:37.754914421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:11:37.75865101Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:11:38.189998253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:39.126394301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:11:39.130237079Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:11:39.296409708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:11:41.89367608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:11:41.897047627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:11:42.55646932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:11:42.559996531Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:11:43.189642423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:11:43.237818956Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:11:43.538195527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:11:43.541532379Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:11:48.189247495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:49.815833398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:11:49.818700877Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:11:50.141461402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:11:50.145638811Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:11:51.3969023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:11:51.402462998Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:11:53.172913985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:11:53.173952069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:11:53.189190823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:11:53.189242413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:11:53.189256216Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T05:11:53.189263174Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:11:53.189266785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:53.189412085Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=debug ts=2026-04-21T05:11:53.190055745Z caller=checkpoint.go:378 msg="writing series" size="444 kB"
level=info ts=2026-04-21T05:11:54.593198265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:11:54.596981632Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:11:57.545480598Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-04-21T05:11:57.765312362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:11:57.769237012Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-04-21T05:11:58.189431719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:11:59.853159527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:11:59.859662905Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:12:01.90130632Z caller=checkpoint.go:378 msg="writing series" size="767 kB"
level=debug ts=2026-04-21T05:12:01.901329993Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-04-21T05:12:03.189237994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:04.694999615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:12:04.700360623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:12:04.70038274Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:12:04.700414051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:12:04.703827288Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:12:06.254358502Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T05:12:07.283655632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:12:07.327152327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:12:07.331510377Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-21T05:12:08.189934232Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:09.318746298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:12:09.322448056Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:12:10.609472698Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T05:12:12.163891633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:12:12.167900179Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:12:13.189730291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:12:13.774302319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:12:14.964542609Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-04-21T05:12:17.929270243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:12:17.932990774Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:12:18.190201307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:19.285539184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:12:19.28973953Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:12:19.319229918Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=info ts=2026-04-21T05:12:19.603070683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:12:19.606950942Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:12:22.654671748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:12:22.659441684Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:12:23.189438293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:23.189459596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:12:23.189478089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:12:23.189486246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:12:23.674077033Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:12:24.611466238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:12:24.615535041Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:12:26.217732996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:12:26.221468789Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:12:28.028986827Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-04-21T05:12:28.18946629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:29.032789881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:12:29.036203234Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:12:30.277239001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:12:30.280930961Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:12:30.321023789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:12:30.324413634Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:12:30.593270071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:12:30.596616047Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:12:31.594496932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:12:31.598012409Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:12:32.383927357Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T05:12:33.189628997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:12:36.73863593Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
ts=2026-04-21T05:12:37.285747264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:12:37.624028132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:12:37.6278236Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:12:38.190202269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:39.19560504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:12:39.198829067Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:12:40.116798788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:12:40.119999058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:12:40.227808086Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:12:41.093545273Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=warn ts=2026-04-21T05:12:43.189968511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:44.346881157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:12:44.352100489Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:12:45.155369106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:12:45.159035821Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:12:45.448338253Z caller=checkpoint.go:378 msg="writing series" size="243 kB"
level=info ts=2026-04-21T05:12:46.149760692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:12:46.154106638Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:12:48.189444365Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:12:48.454441385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:12:48.458424078Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:12:48.729745526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:12:49.803417137Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=debug ts=2026-04-21T05:12:49.803441015Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T05:12:52.742263454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:12:52.746600898Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:12:53.172860191Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:12:53.173900047Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:12:53.189173589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:12:53.189202231Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:12:53.189257071Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:12:53.189267644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:12:54.158435065Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
ts=2026-04-21T05:12:55.19136426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:58638"
level=warn ts=2026-04-21T05:12:58.190244661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:12:58.513440232Z caller=checkpoint.go:378 msg="writing series" size="449 kB"
level=info ts=2026-04-21T05:12:59.032915615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:12:59.039734758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:13:01.775864077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:13:01.779340017Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:13:02.867359536Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T05:13:03.190221589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:03.677335486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:13:03.681283953Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:13:04.991895791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:13:04.996263364Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:13:07.222253412Z caller=checkpoint.go:378 msg="writing series" size="821 B"
ts=2026-04-21T05:13:07.287808868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=info ts=2026-04-21T05:13:07.751339252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:13:07.756804892Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:13:08.189494683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:09.269702959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:13:09.273301321Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:13:09.696321991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:13:09.699817617Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:13:11.577412939Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-21T05:13:13.189557299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:13.738243945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:13:13.741501874Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:13:14.780787458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:13:14.782334285Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T05:13:15.700173914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:13:15.703283868Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:13:15.931975587Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=info ts=2026-04-21T05:13:16.512878573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:13:16.516303119Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:13:18.190169509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:19.172259295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:13:19.176102802Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:13:20.287293987Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T05:13:23.189753594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:13:23.189765293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:13:23.189779881Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:13:23.189788211Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:13:23.682030245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:13:23.685871724Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:13:24.642203684Z caller=checkpoint.go:378 msg="writing series" size="374 kB"
level=debug ts=2026-04-21T05:13:24.649201249Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:13:26.032348628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="405 B" avg_comp="405 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:13:26.035882618Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:13:28.189889179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:28.231037025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:13:28.235034432Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:13:28.997977828Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T05:13:28.99800113Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=9
level=info ts=2026-04-21T05:13:30.434678111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:13:30.438909887Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:13:32.923226734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:13:32.92720117Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:13:33.058692705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:13:33.062471335Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:13:33.189955515Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:13:33.351351654Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
level=info ts=2026-04-21T05:13:34.516257426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:13:34.522080762Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:13:34.522106169Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:13:34.522133896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:13:34.526900047Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:13:34.668506495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:13:34.671831345Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:13:35.187553951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:13:35.190923953Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:13:36.059235084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:13:36.063073635Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-21T05:13:37.290879385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=debug ts=2026-04-21T05:13:37.706365558Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T05:13:38.189781104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:13:42.061388994Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T05:13:43.18961808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:13:44.394301348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35272"
level=info ts=2026-04-21T05:13:44.988269711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:13:44.997113675Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:13:46.415830275Z caller=checkpoint.go:378 msg="writing series" size="252 kB"
level=warn ts=2026-04-21T05:13:48.189568901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:49.600732698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:13:49.604274607Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:13:50.77069864Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-04-21T05:13:51.715913249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:13:51.721468682Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:13:52.942549281Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:13:53.173168193Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:13:53.173182281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:13:53.189486056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:13:53.189485123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:13:53.189512296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:13:53.189520902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:13:54.326115612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:13:54.330142666Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:13:54.608945465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:13:54.612385081Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:13:55.125915563Z caller=checkpoint.go:378 msg="writing series" size="342 kB"
ts=2026-04-21T05:13:55.194991987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:55068"
level=debug ts=2026-04-21T05:13:55.799925786Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:13:58.189618939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:13:58.955775448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:13:58.959458473Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:13:59.480306742Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=info ts=2026-04-21T05:13:59.988712463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:13:59.992057559Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:14:02.568531429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:14:02.571929997Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:14:02.828470987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:14:02.832346413Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:14:03.190095432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:14:03.835786111Z caller=checkpoint.go:378 msg="writing series" size="4.7 kB"
level=info ts=2026-04-21T05:14:06.586504277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:14:06.590008761Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-04-21T05:14:07.293506589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:14:08.189621493Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:14:08.189955888Z caller=checkpoint.go:378 msg="writing series" size="535 kB"
level=debug ts=2026-04-21T05:14:08.189971586Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-04-21T05:14:08.755126666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:14:08.758418602Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:14:10.424962622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:14:10.428344993Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:14:12.544790466Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=warn ts=2026-04-21T05:14:13.189618796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:14:15.89808398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:57914"
level=info ts=2026-04-21T05:14:16.589548375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:14:16.594932715Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:14:16.899775349Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=info ts=2026-04-21T05:14:17.370463985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:14:17.375012585Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T05:14:18.189501009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:14:19.540652748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:14:19.54439904Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:14:20.227616826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:14:21.254583393Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:14:22.107597602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:14:22.112711524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:14:22.300404744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:14:22.304046563Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:14:23.189978329Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:14:23.190001446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:14:23.190010276Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:14:23.189974984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:14:23.257155228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:14:23.261116217Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:14:24.854362649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:14:24.85894595Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T05:14:25.609653445Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:14:26.240602655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:14:26.244123048Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:14:28.1901177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:14:28.268489878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:14:28.271985202Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:14:29.964397503Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T05:14:32.247646272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:14:32.251140354Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:14:32.676395958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:14:32.679943948Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:14:33.190234977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:14:34.318881612Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T05:14:35.329561059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:14:35.3331803Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:14:36.642189633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:14:36.649479569Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T05:14:37.296894289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:14:38.189791428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:14:38.673868268Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T05:14:39.941156616Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:14:42.291604196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:14:42.295127631Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:14:43.029082017Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-21T05:14:43.189484812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:14:43.573807122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:14:43.577137087Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:14:45.265085068Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:14:46.915592847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:14:46.922336071Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:14:47.384012686Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-21T05:14:48.189562487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:14:50.012775604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:14:50.016304034Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:14:50.67160783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:14:50.67511712Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:14:51.739138337Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
level=info ts=2026-04-21T05:14:52.807277129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:14:52.811242968Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:14:52.811270214Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:14:52.811297073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:14:52.814492525Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:14:53.173052292Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:14:53.17353105Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:14:53.189784688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:14:53.189810396Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:14:53.189819802Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:14:53.18981417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:14:56.093319089Z caller=checkpoint.go:378 msg="writing series" size="197 kB"
level=info ts=2026-04-21T05:14:57.311223458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:14:57.315840265Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:14:57.458269544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:14:57.461887343Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T05:14:58.189386874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:00.448597621Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T05:15:03.189739648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:15:03.801195413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:15:03.805033319Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:15:04.803558904Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
ts=2026-04-21T05:15:07.299328899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:15:08.19000534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:09.158295834Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=info ts=2026-04-21T05:15:09.796948148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:15:09.801011292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:15:11.369518886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:15:11.373038294Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T05:15:13.189788802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:13.513055908Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:15:13.872616722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:15:13.876306799Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T05:15:15.901735577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:53866"
level=info ts=2026-04-21T05:15:17.113672119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:15:17.11874078Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:15:17.867171296Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:15:18.189482273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:15:19.059061148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:15:19.062739632Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:15:19.06276217Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:15:19.062790343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:15:19.065863789Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:15:19.246430741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:15:19.251425769Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:15:19.274821692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:15:20.635671129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:15:20.639194411Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:15:21.104497077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:15:21.107650275Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:15:21.963256399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:15:21.967058829Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:15:22.22293896Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:15:23.18925702Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:15:23.189283368Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:15:23.189292279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:15:23.189284146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:15:24.115804075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:15:24.119903042Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:15:24.12394899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="465 kB" avg_comp="465 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:15:24.127284989Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dae343f19%3A19dae57db80%3A953a75cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T05:15:24.127298676Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:15:24.127320764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:15:24.130419857Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:15:26.577507411Z caller=checkpoint.go:378 msg="writing series" size="881 kB"
level=debug ts=2026-04-21T05:15:26.577529268Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=18
level=debug ts=2026-04-21T05:15:27.196687246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:15:28.189338776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:30.944512677Z caller=checkpoint.go:378 msg="writing series" size="15 MB"
level=debug ts=2026-04-21T05:15:30.944538772Z caller=checkpoint.go:390 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-04-21T05:15:31.54328005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:15:31.54800909Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:15:33.189449968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:15:34.478834692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:15:34.483362846Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:15:34.621943535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:15:34.625448704Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-04-21T05:15:34.625469427Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:15:34.625495216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:15:34.628574984Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:15:35.014652706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:15:35.021256844Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:15:35.121487049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:15:35.125083031Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:15:35.286847204Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
ts=2026-04-21T05:15:37.301552397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=info ts=2026-04-21T05:15:37.765489906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:15:37.769054003Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:15:38.189273137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:15:38.235391008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:15:38.238603977Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:15:39.289333158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:15:39.292582467Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:15:39.303794658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:15:39.307816125Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:15:39.641971948Z caller=checkpoint.go:378 msg="writing series" size="7.4 kB"
level=warn ts=2026-04-21T05:15:43.189853136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:43.996791162Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
ts=2026-04-21T05:15:44.402738322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39720"
level=info ts=2026-04-21T05:15:45.446688381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:15:45.450550376Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T05:15:48.189557984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:48.352048446Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-04-21T05:15:48.472680297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:15:48.477689979Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:15:48.477709626Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:15:48.477734179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:15:48.480466078Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:15:48.974119282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:15:48.977849735Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:15:50.36017846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:15:50.363778906Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:15:51.918293451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:15:52.706737575Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-04-21T05:15:53.172603524Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:15:53.173669269Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:15:53.174965732Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T05:15:53.175004091Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974164 n=1
level=debug ts=2026-04-21T05:15:53.175045897Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974164 n=1
level=info ts=2026-04-21T05:15:53.190001493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:15:53.190030106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:15:53.190038533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:15:53.190047642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:15:57.061283492Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T05:15:58.189920768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:16:01.416418952Z caller=checkpoint.go:378 msg="writing series" size="6.2 kB"
level=info ts=2026-04-21T05:16:03.106484041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:16:03.111672933Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:16:03.18926294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:04.811263202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:16:04.815014404Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:16:04.815034383Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:04.815055293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:16:04.817722009Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:16:05.771383399Z caller=checkpoint.go:378 msg="writing series" size="341 kB"
level=info ts=2026-04-21T05:16:06.636012834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:16:06.640066804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-04-21T05:16:06.640087754Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:06.640110964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:16:06.643107781Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-04-21T05:16:07.304450187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=info ts=2026-04-21T05:16:07.367088829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:16:07.370179531Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:16:07.684955018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:16:07.688425933Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:16:07.982346579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:16:07.986158366Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:16:08.19001688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:16:10.125819033Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=debug ts=2026-04-21T05:16:10.125843907Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-21T05:16:12.234926655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:16:12.240014364Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:16:13.189827052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:16:14.416681408Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:16:14.480916383Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T05:16:14.667735857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:16:14.671337884Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:16:15.108457376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:16:15.11167912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:16:15.111701198Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:15.111726348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:16:15.114503511Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:16:16.016523532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:16:16.019938836Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T05:16:16.019959009Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:16.019981763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:16:16.022797249Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:16:18.189751781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:18.302055289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:16:18.306901442Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:16:18.523098559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:16:18.530628598Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:16:18.530652992Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:18.530683989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:16:18.53740049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:16:18.835282931Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=info ts=2026-04-21T05:16:19.155136737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:16:19.15851634Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:16:19.290232044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:16:19.293580678Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-04-21T05:16:19.29360092Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:19.293623113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:16:19.296364013Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:16:20.173954422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:16:20.177377933Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:16:20.177400952Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:20.177427544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:16:20.180245852Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:16:20.55918362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:16:20.562625892Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:16:22.160761206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:16:22.164111352Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:16:22.638667232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:16:22.642271558Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:16:22.642291803Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:22.642313734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:16:22.645044847Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:16:23.190015967Z caller=checkpoint.go:390 msg="flushing series" totalSize="106 kB" series=2
level=warn ts=2026-04-21T05:16:23.190062729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:23.190082041Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:16:23.19009646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:16:23.190104995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:16:23.19580584Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-04-21T05:16:23.197680542Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008142387s
level=debug ts=2026-04-21T05:16:23.875582451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:16:24.34402531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:16:24.348072199Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:16:25.174769628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:16:25.178240352Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:16:25.178272121Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:25.178296594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:16:25.181203741Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:16:28.189937245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:29.671970836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:16:29.675421163Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:16:29.675440183Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:29.675462455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:16:29.678140312Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:16:30.061142639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:16:30.064599796Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:16:30.693511361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:16:30.696679089Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:16:31.12383631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:16:31.127315964Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:16:32.593721166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:16:32.599700033Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:16:33.189516625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:33.743069047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:16:33.746666246Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:16:33.746692044Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:33.746718786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:16:33.749781348Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:16:36.410496998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:16:36.418057341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T05:16:37.307882393Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:16:38.189840413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:39.120526212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:16:39.123977317Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:16:39.849579986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:16:39.85301061Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:16:42.113163462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:16:42.116774087Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:16:42.253333533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:16:42.256859837Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:16:42.393428807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:16:42.397007271Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:16:42.685741761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:16:42.689261959Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:16:43.190193595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:43.267474509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:16:43.271076822Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:16:43.271095704Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:43.271120261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:16:43.273874763Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T05:16:45.9093354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:55688"
level=warn ts=2026-04-21T05:16:48.189854578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:50.110854498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:16:50.116025332Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:16:52.873323463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:16:52.876726767Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:16:53.173058424Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:16:53.173545005Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:16:53.189821708Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T05:16:53.189842806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:16:53.189866441Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:16:53.18987549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:16:53.189879127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:16:53.189958749Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=debug ts=2026-04-21T05:16:53.190309254Z caller=checkpoint.go:378 msg="writing series" size="472 kB"
level=info ts=2026-04-21T05:16:53.521514292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:16:53.526403432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:16:54.192630963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T05:16:55.208120089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:42418"
level=info ts=2026-04-21T05:16:55.793321208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:16:55.796683338Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:16:55.969329509Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:16:55.971176929Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T05:16:56.910794793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:16:56.91380265Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:16:57.49863631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:16:57.5021877Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:16:57.545281387Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-04-21T05:16:58.004738226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:16:58.007882916Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:16:58.007901465Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:16:58.007920968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:16:58.010456337Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:16:58.189710218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:17:01.901179869Z caller=checkpoint.go:378 msg="writing series" size="927 kB"
level=debug ts=2026-04-21T05:17:01.901234427Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-04-21T05:17:03.1896548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:03.325322543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:17:03.329819946Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:17:03.483660883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:17:03.487593676Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:17:03.955897177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:17:03.95964862Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:17:04.014019563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:17:04.017520304Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:17:05.083649551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:17:05.087185714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:17:06.255702534Z caller=checkpoint.go:378 msg="writing series" size="654 B"
level=info ts=2026-04-21T05:17:06.841927825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:17:06.845981147Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-21T05:17:07.309592167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:17:08.189240504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:08.726328234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:17:08.729943743Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:17:10.609976876Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-04-21T05:17:10.942916624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:17:10.946520415Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-21T05:17:13.189801283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:14.434918356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:17:14.439005894Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:17:14.966339753Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=info ts=2026-04-21T05:17:15.851823266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:17:15.855371286Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:17:16.076110316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:17:16.083466363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:17:16.81208551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:17:16.81573873Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:17:16.815764194Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:17:16.815788702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:17:16.818742993Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:17:18.189965695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:18.195098981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:17:18.198508801Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:17:19.311697822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:17:19.31502223Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:17:19.320177149Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=debug ts=2026-04-21T05:17:21.754433424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:17:23.189457655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:17:23.189485895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:17:23.189493977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:17:23.189496632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:23.38373454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:17:23.388048943Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:17:23.674230734Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:17:23.966829658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:17:23.97053155Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T05:17:25.210906949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:52444"
level=debug ts=2026-04-21T05:17:28.029373256Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-04-21T05:17:28.190202124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:29.811349939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:17:29.815518894Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:17:32.384464497Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T05:17:33.189825268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:17:36.739460261Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-04-21T05:17:37.210070352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:17:37.214987201Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-21T05:17:37.311437175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:17:38.190063963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:38.773734309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:17:38.777520326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:17:39.416342785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:17:39.420179124Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:17:39.727854257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:17:39.73113155Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:17:41.093822298Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=debug ts=2026-04-21T05:17:43.047184443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:17:43.190115669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:17:44.409924598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60864"
level=debug ts=2026-04-21T05:17:45.44917502Z caller=checkpoint.go:378 msg="writing series" size="243 kB"
level=warn ts=2026-04-21T05:17:48.190016332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:49.106062727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:17:49.109620916Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:17:49.803811553Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=debug ts=2026-04-21T05:17:49.803836973Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-04-21T05:17:52.143403674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:17:52.146941705Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:17:52.2636148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:17:52.268648058Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:17:53.173042827Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:17:53.173240983Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:17:53.18945358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:17:53.189480747Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:17:53.189488451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:17:53.189505222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:17:53.394233582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:17:53.399790768Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:17:53.705796722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:17:53.709172598Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:17:54.118961089Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:17:54.158126769Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=info ts=2026-04-21T05:17:55.708838715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:17:55.712305027Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:17:58.189676574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:17:58.513045409Z caller=checkpoint.go:378 msg="writing series" size="592 kB"
level=info ts=2026-04-21T05:17:59.330743869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:17:59.338273346Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:18:00.092907077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:18:00.096845952Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:18:00.937425899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:18:00.940940552Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:18:01.837665593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:18:01.841621191Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:18:02.241542162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:18:02.244870891Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:18:02.86762469Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T05:18:03.190088905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:18:03.372898735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:18:03.376342274Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:18:04.366248193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:18:04.373726251Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:18:07.222608295Z caller=checkpoint.go:378 msg="writing series" size="821 B"
ts=2026-04-21T05:18:07.314056816Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:18:08.189476679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:11.577851785Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-21T05:18:11.785481366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:18:11.790336507Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:18:13.189197474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:18:14.413018036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52002"
level=info ts=2026-04-21T05:18:14.623662939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:18:14.627814042Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:18:15.083141487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:18:15.086700288Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:18:15.933125492Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-21T05:18:18.189632903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:19.341469077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:18:19.60438485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:18:19.607693283Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:18:20.287763507Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=debug ts=2026-04-21T05:18:20.504635712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:18:21.767860197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:18:21.771600647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:18:23.189386577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:18:23.189417404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:18:23.189429031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:18:23.189437988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:24.642789581Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=info ts=2026-04-21T05:18:27.591263705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:18:27.595540293Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:18:28.18947396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:28.99796288Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-21T05:18:28.997986179Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=9
level=info ts=2026-04-21T05:18:30.315067413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:18:30.319502544Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:18:33.189255638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:33.351769672Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=info ts=2026-04-21T05:18:34.954117035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:18:34.958359625Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:18:36.114739068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:18:36.11864323Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:18:37.137403278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:18:37.140866898Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-21T05:18:37.317596251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:18:37.608856123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:18:37.612190167Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:18:37.70632046Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T05:18:38.189365555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:42.061266068Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-04-21T05:18:42.092401776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:18:42.098114902Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:18:43.189855892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:18:44.4154173Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56456"
level=info ts=2026-04-21T05:18:44.811940422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:18:44.815658644Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:18:45.518621645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:18:45.521932864Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:18:46.417120094Z caller=checkpoint.go:378 msg="writing series" size="305 kB"
level=debug ts=2026-04-21T05:18:46.901538861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:18:47.140167968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:18:47.143833419Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:18:47.795514264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:18:47.799550008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:18:47.799573559Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:18:47.799601086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:18:47.803454846Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:18:48.189507704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:50.771252771Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-21T05:18:51.097082412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:18:51.101050858Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:18:51.870782303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:18:51.874424136Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:18:52.895259005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="405 B" avg_comp="405 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:18:52.898836523Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:18:53.172708212Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:18:53.173769396Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:18:53.19007071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:18:53.190100023Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:18:53.190108199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:18:53.190120631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:18:55.126192169Z caller=checkpoint.go:378 msg="writing series" size="421 kB"
level=debug ts=2026-04-21T05:18:55.126242231Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-04-21T05:18:55.220458637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56500"
level=info ts=2026-04-21T05:18:57.071113174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:18:57.079041227Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:18:57.248704465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:18:57.252171691Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:18:57.382721708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:18:57.386012719Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:18:58.190051913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:18:59.362420545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:18:59.367884568Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:18:59.481533043Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=info ts=2026-04-21T05:19:02.7752428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:19:02.781173233Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:19:03.189662996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:19:03.835746978Z caller=checkpoint.go:378 msg="writing series" size="5.0 kB"
ts=2026-04-21T05:19:07.320456201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:19:08.189670921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:19:08.190928279Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=debug ts=2026-04-21T05:19:08.971183201Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:19:09.617702673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:19:09.621125701Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:19:11.688000195Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:19:12.493950796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:19:12.497995737Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:19:12.545266448Z caller=checkpoint.go:378 msg="writing series" size="168 kB"
level=warn ts=2026-04-21T05:19:13.189408211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:19:13.856238275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:19:13.859810537Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-21T05:19:14.418741341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46794"
level=info ts=2026-04-21T05:19:15.392139016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:19:15.395538673Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:19:16.900896596Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=info ts=2026-04-21T05:19:17.945941253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:19:17.949657214Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T05:19:18.189475663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:19:21.255624064Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:19:23.189908769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:19:23.189934595Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:19:23.189942546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:19:23.189944407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:19:23.961918148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:19:23.965774087Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:19:24.064033036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:19:24.068053216Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:19:25.611316199Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:19:26.949984774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:19:26.953965499Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:19:27.855628574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:19:27.859408835Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:19:28.104195643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:19:28.107978766Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:19:28.189669443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:19:29.964736212Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-21T05:19:33.189386279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:19:33.479493873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:19:33.484384955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:19:34.320176376Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T05:19:37.128230776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:19:37.131826923Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-04-21T05:19:37.323623703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:19:38.189371215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:19:38.674725283Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T05:19:38.846766109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:19:38.85187468Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:19:39.964784266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:19:39.968224497Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:19:39.984327817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:19:41.154795605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:19:41.158792263Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:19:42.748336357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:19:42.752527836Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:19:43.029496858Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-21T05:19:43.169344549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:19:43.172612917Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:19:43.189975346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:19:43.390887309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:19:43.394932221Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-04-21T05:19:44.421433467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48432"
level=info ts=2026-04-21T05:19:46.091322242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:19:46.094854698Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:19:47.38392396Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-21T05:19:48.189866187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:19:51.739155755Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=info ts=2026-04-21T05:19:52.369119701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:19:52.372682201Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:19:52.759696359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:19:52.767152963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:19:53.173030065Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:19:53.173468158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:19:53.189774161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:19:53.189802676Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:19:53.189814238Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:19:53.189812898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:19:55.838836785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:19:55.846926811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:19:56.093609613Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-04-21T05:19:56.254422165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:19:56.257876189Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:19:58.189462608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:00.448612839Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=debug ts=2026-04-21T05:20:00.448637311Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=15
level=info ts=2026-04-21T05:20:01.751338791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:20:01.755307668Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:20:03.190162817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:04.803686279Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-21T05:20:06.517313218Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:20:07.272020177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:20:07.275362944Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T05:20:07.32659088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:20:08.190007013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:09.15824488Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=debug ts=2026-04-21T05:20:09.219361834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:20:09.825026919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:20:09.828850684Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:20:13.18939831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:13.513467664Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:20:14.41562636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:20:14.419634803Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T05:20:14.423997295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54154"
level=info ts=2026-04-21T05:20:17.215321424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:20:17.219111376Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:20:17.276310131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:20:17.279920375Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:20:17.867595265Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:20:18.189245454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:20:18.965321476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:20:18.968668006Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:20:20.757339357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:20:20.761024677Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:20:22.223302246Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:20:22.880337892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:20:22.884148414Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:20:23.190059461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:20:23.190090133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:20:23.19009952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:20:23.19013174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:26.578531605Z caller=checkpoint.go:378 msg="writing series" size="982 kB"
level=debug ts=2026-04-21T05:20:26.578559497Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-04-21T05:20:27.772438502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:20:27.776006467Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:20:28.189387663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:20:30.845494968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:20:30.849227592Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:20:30.944506351Z caller=checkpoint.go:378 msg="writing series" size="16 MB"
level=debug ts=2026-04-21T05:20:30.944535346Z caller=checkpoint.go:390 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-04-21T05:20:32.058925835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:20:32.062419996Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:20:32.747784099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:20:32.751825419Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:20:33.189236538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:20:33.84810623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:20:33.851713801Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:20:35.181033306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:20:35.184801826Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:20:35.287234005Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-04-21T05:20:36.116451402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:20:36.121442906Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:20:36.121464569Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:20:36.121490003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:20:36.126237445Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T05:20:37.32876412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T05:20:37.352568024Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:20:37.388937613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:20:37.393072304Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:20:38.013692338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:20:38.01797498Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-21T05:20:38.189704656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:39.642642116Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=warn ts=2026-04-21T05:20:43.189292729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:43.997588312Z caller=checkpoint.go:378 msg="writing series" size="249 kB"
level=info ts=2026-04-21T05:20:45.467730888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:20:45.475132431Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T05:20:45.931743458Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:42174"
level=info ts=2026-04-21T05:20:45.993548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:20:45.997160026Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-04-21T05:20:48.189438129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:48.351912557Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-04-21T05:20:49.084689539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:20:49.088319599Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:20:49.517489585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:20:49.521089947Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:20:49.8349631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:20:49.840693346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:20:51.930331188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:20:51.934043689Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:20:52.706948171Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-04-21T05:20:52.86925794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:20:52.871108837Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T05:20:53.172733636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:20:53.173784179Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:20:53.190122438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:20:53.19015481Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:20:53.190162823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:20:53.190178896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:20:57.061529822Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T05:20:57.317374852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:20:57.320926712Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:20:58.189753836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:21:01.416906509Z caller=checkpoint.go:378 msg="writing series" size="6.2 kB"
level=info ts=2026-04-21T05:21:01.881876511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:21:01.886197472Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:21:03.18977402Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:21:05.771842131Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
ts=2026-04-21T05:21:07.330037602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:21:08.189650415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:21:09.116230994Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:21:09.300740228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:21:09.30788174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:21:09.603718471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:21:09.607892459Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:21:10.126304711Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T05:21:12.621707523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:21:12.625405924Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:21:12.866116755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:21:12.869962301Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:21:13.189201178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:13.839258418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:21:13.843351645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T05:21:14.427828159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38820"
level=debug ts=2026-04-21T05:21:14.480511328Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T05:21:14.91877875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:21:14.922353609Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T05:21:15.933166794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:59368"
level=warn ts=2026-04-21T05:21:18.189481494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:18.344839132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:21:18.349435724Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:21:18.349458812Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:21:18.349482618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:21:18.35242088Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:21:18.701137792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:21:18.704619701Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:21:18.836099936Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=debug ts=2026-04-21T05:21:18.836123837Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T05:21:19.65533066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:21:19.658880229Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:21:19.963743214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:21:19.969120589Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:21:20.883373606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:21:20.886904396Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:21:22.599036949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:21:22.602716198Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:21:22.802240954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:21:22.806339297Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:21:23.189907335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:21:23.189937788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:21:23.189947118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:21:23.189978468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:23.19827416Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-04-21T05:21:23.200456343Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010368056s
ts=2026-04-21T05:21:25.233429286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36610"
level=debug ts=2026-04-21T05:21:27.60049363Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:21:27.658736428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:21:27.662506687Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:21:28.190165907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T05:21:33.189585341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:33.75906213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:21:33.762895672Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:21:34.554124829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:21:34.562972757Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:21:35.510614596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:21:35.513941111Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-21T05:21:37.331450821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:21:38.190022149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:40.554012632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:21:40.558979984Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:21:43.189754424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:21:44.212108168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T05:21:44.43107168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39980"
level=info ts=2026-04-21T05:21:45.712277095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:21:45.716487896Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-04-21T05:21:45.934825496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:44314"
level=info ts=2026-04-21T05:21:47.606564455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:21:47.610529478Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:21:48.190031795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:48.995923718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:21:49.000804326Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:21:53.130943054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:21:53.135233128Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:21:53.173045228Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:21:53.173471929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:21:53.189746604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T05:21:53.189762135Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-21T05:21:53.189773861Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:21:53.189784606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:21:53.189797828Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:53.189896306Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=debug ts=2026-04-21T05:21:53.190306478Z caller=checkpoint.go:378 msg="writing series" size="480 kB"
level=info ts=2026-04-21T05:21:55.902758097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:21:55.906227091Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:21:57.545535118Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=warn ts=2026-04-21T05:21:58.189829059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:21:59.637512522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:21:59.641142132Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:21:59.641167303Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:21:59.641201745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:21:59.644683269Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:22:01.818907647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:22:01.822562077Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:22:01.901327302Z caller=checkpoint.go:378 msg="writing series" size="935 kB"
level=debug ts=2026-04-21T05:22:01.901352286Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-04-21T05:22:02.355140912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:22:02.358459175Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-04-21T05:22:03.189744159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:03.838021534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:22:03.841980685Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:22:04.974492997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:22:05.220250249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:22:05.223574032Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:22:06.254988471Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T05:22:07.332992106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:22:08.18987873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:08.641259465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:22:08.644681434Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:22:09.178006035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:22:09.181349063Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:22:09.185397084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:22:09.188884186Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:22:09.188901392Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:09.188924986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:22:09.192247936Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:22:09.268499625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:22:09.271708567Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:22:10.610383776Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-21T05:22:13.168423467Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:22:13.189776481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:14.357307662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:22:14.362276045Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:22:14.964977559Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:22:15.230108831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:22:15.235129904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:22:16.893346174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:22:16.896690119Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:22:18.190131625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:18.411966829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:22:18.4153808Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:22:19.3200475Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=info ts=2026-04-21T05:22:19.410354088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:22:19.42143682Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-04-21T05:22:19.421463446Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:19.421491446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:22:19.424723697Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:22:19.846875517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:22:19.850237138Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:22:19.850263145Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:19.850290153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:22:19.853014548Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T05:22:23.189302719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:23.189312748Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:22:23.189330468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:22:23.189338681Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:22:23.674198076Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:22:25.487231921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:22:25.491010995Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:22:25.491035886Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:25.4910625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:22:25.494169674Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:22:28.029784771Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-04-21T05:22:28.189386657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:28.588195893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:22:28.591740616Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:22:32.384637975Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=info ts=2026-04-21T05:22:32.555971465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:22:32.563549156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:22:33.177524709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:22:33.180967098Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T05:22:33.189263268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:36.417425401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:22:36.420758648Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:22:36.420781708Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:36.420809007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:22:36.423560506Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:22:36.739511068Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
ts=2026-04-21T05:22:37.335551045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:22:38.189686477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:39.2653137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:22:39.269260725Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:22:41.093651521Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-21T05:22:43.190178296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:43.944258298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:22:43.948042736Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:22:44.410651382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:22:44.415363315Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-04-21T05:22:44.415385948Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:44.415410206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:22:44.418887908Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:22:44.770089731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:22:44.773323708Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:22:45.448277616Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=debug ts=2026-04-21T05:22:47.064877509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:22:47.803955827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:22:47.807230999Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:22:47.885346382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:22:47.88854196Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-04-21T05:22:47.888566657Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:47.888586302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:22:47.891262184Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:22:48.003635657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:22:48.006886512Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:22:48.189655026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:22:48.198809353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:22:48.201798121Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:22:48.20181253Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:48.201832831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:22:48.204490273Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:22:49.559492217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:22:49.563685198Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:22:49.803698289Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=debug ts=2026-04-21T05:22:49.80372478Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=debug ts=2026-04-21T05:22:49.899963402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:22:50.309269804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:22:50.31301131Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:22:50.675263913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:22:50.678716496Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:22:50.678739117Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:50.678768732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:22:50.681460241Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:22:50.720003678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:22:50.723582233Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:22:51.887697588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:22:51.89190418Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:22:53.172616394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:22:53.173666876Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:22:53.189965993Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:22:53.18999425Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:22:53.190004468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:22:53.190025114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:22:54.158404484Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-04-21T05:22:55.645051956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:22:55.650238824Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:22:56.815380888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:22:56.818737919Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T05:22:56.81875718Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:22:56.818780656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:22:56.821557569Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:22:58.189466302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:22:58.513716425Z caller=checkpoint.go:378 msg="writing series" size="498 kB"
level=info ts=2026-04-21T05:22:59.736531799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:22:59.739810307Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:23:01.507971612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:23:01.511548104Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:23:01.511568506Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:01.511589924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:23:01.514390941Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:23:02.520271247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:23:02.523567039Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:23:02.867680024Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T05:23:03.189692386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:23:03.787619427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:23:03.790993762Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:23:05.680240028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:23:05.683800792Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:23:07.223235685Z caller=checkpoint.go:378 msg="writing series" size="821 B"
level=info ts=2026-04-21T05:23:07.321519686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:23:07.325758008Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T05:23:07.339040934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:23:08.189865798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:23:08.670479445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:23:08.674113791Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:23:08.674134565Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:08.674158627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:23:08.676883162Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:23:09.136130484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:23:09.139631123Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:23:09.139650654Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:09.139672454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:23:09.142540798Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:23:10.950951997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:23:10.954380973Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:23:10.965569738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:23:10.968876102Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:23:11.577832297Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-21T05:23:11.643975575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:23:11.647468382Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:23:13.189309818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:23:14.440366519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55480"
level=debug ts=2026-04-21T05:23:15.93260168Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-21T05:23:18.190306336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:23:18.749459959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:23:18.7606211Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:23:19.067603421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:23:19.071188948Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:23:19.992037709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:23:19.996813992Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:23:20.287372049Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T05:23:21.399129506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:23:21.402514863Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:23:22.228578596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:23:23.189480265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:23:23.189503064Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T05:23:23.189494952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:23:23.18951062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:23:23.790365337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:23:23.797273878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:23:23.797298783Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:23.797328694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:23:23.803639205Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:23:24.642200486Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-04-21T05:23:25.741258835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:23:25.744646061Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:23:26.137051712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:23:26.140478516Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:23:27.505842188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:23:27.509092299Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:23:28.189947935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:23:28.998139727Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T05:23:28.998162845Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=9
level=info ts=2026-04-21T05:23:30.857773626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:23:30.861354489Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:23:31.532762036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:23:31.53649806Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:23:33.190142903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:23:33.352458209Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
level=info ts=2026-04-21T05:23:34.843498426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:23:34.847025895Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:23:36.243100681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="405 B" avg_comp="405 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:23:36.246612866Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:23:36.246633104Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:36.246655497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:23:36.24930793Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T05:23:37.341323488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T05:23:37.706946322Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T05:23:38.189587029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:23:39.347097231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:23:39.351099252Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:23:41.518265134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:23:41.525601778Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:23:42.061366109Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=info ts=2026-04-21T05:23:42.179799955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:23:42.184053257Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:23:43.189462408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:23:43.920361535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:23:43.923863334Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:23:46.41667888Z caller=checkpoint.go:378 msg="writing series" size="368 kB"
level=info ts=2026-04-21T05:23:46.886845636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:23:46.890683232Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:23:46.890706425Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:46.890730905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:23:46.893673333Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:23:47.289698357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:23:47.293645254Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:23:47.882640706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:23:47.885925359Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:23:48.189257905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:23:48.190478631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:23:48.737721195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:23:48.740909394Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:23:49.353054197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:23:49.360476671Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:23:50.770863401Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-04-21T05:23:52.947430534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:23:52.951926008Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:23:53.172879045Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:23:53.173926448Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:23:53.189165841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:23:53.189197945Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:23:53.189225812Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:23:53.189196399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:23:55.126402501Z caller=checkpoint.go:378 msg="writing series" size="485 kB"
level=debug ts=2026-04-21T05:23:55.126427626Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-04-21T05:23:55.566708054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:23:55.570186431Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:23:55.570239777Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:23:55.570263918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:23:55.573155971Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:23:55.588324257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:23:55.593380491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:23:55.598273432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:23:58.189997812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:23:59.480733514Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=info ts=2026-04-21T05:23:59.755375634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:23:59.759083244Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:24:01.13448493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:24:01.138365944Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:24:01.730362538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:24:01.734060494Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-21T05:24:03.189323424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:24:03.835989232Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-04-21T05:24:04.726404619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:24:04.730138169Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:24:05.759968716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:24:05.763842954Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:24:07.215884812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:24:07.220581682Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T05:24:07.343142233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:24:07.511329746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:24:07.514774061Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:24:08.19018249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:24:08.190381699Z caller=checkpoint.go:378 msg="writing series" size="461 kB"
level=info ts=2026-04-21T05:24:09.191364036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:24:09.194693709Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:24:12.545563589Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=warn ts=2026-04-21T05:24:13.189941405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:24:13.664529889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:24:13.671171311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T05:24:15.947316501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:41956"
level=debug ts=2026-04-21T05:24:16.899772652Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=info ts=2026-04-21T05:24:17.754252062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:24:17.758062572Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:24:18.189663665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:24:21.254897153Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:24:22.190195592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:24:22.19542303Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-21T05:24:23.189347471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:24:23.189347046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:24:23.1893798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:24:23.189390996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:24:23.964730034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:24:23.968294524Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:24:24.810775331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:24:24.814589935Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:24:24.986327407Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:24:25.610130996Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:24:26.738968932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:24:26.742376308Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:24:26.788712373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:24:26.791914554Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:24:28.190160384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:24:28.87865589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:24:28.882699642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:24:29.395724605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:24:29.399067249Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:24:29.964508727Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T05:24:32.84017847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:24:32.844240586Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:24:33.189547577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:24:34.320184311Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T05:24:34.508716488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:24:34.512746876Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:24:34.908334148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:24:34.911704498Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T05:24:37.345699373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:24:38.189856633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:24:38.585941553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:24:38.589281064Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:24:38.67456864Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-21T05:24:38.755682542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:24:38.757851832Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T05:24:40.306414836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:24:40.310036444Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:24:40.937712638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:24:40.942603175Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:24:42.10484067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:24:42.108277348Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:24:43.029760549Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-21T05:24:43.189427611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:24:44.445717075Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57142"
level=info ts=2026-04-21T05:24:44.519221687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:24:44.522881894Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:24:47.384249705Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-04-21T05:24:47.625759915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:24:47.629629016Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:24:47.925344742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:24:47.928843723Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:24:48.189643681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:24:51.738887082Z caller=checkpoint.go:378 msg="writing series" size="141 kB"
level=debug ts=2026-04-21T05:24:53.172966506Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:24:53.174015383Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:24:53.189285041Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:24:53.189295938Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T05:24:53.189286111Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:24:53.189303149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:24:53.367225259Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:24:56.094116715Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=debug ts=2026-04-21T05:24:56.094141325Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-04-21T05:24:56.177469277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:24:56.182725302Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:24:58.189683256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:00.449094131Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-21T05:25:03.189967646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:04.803699479Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T05:25:04.877991945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:25:04.881671947Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-21T05:25:07.347310602Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:25:07.392250688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:25:07.393328616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:25:07.396031706Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:25:07.400493052Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T05:25:08.190024739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:09.158584361Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=info ts=2026-04-21T05:25:11.589463387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:25:11.595063153Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:25:13.189305976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:13.513347518Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:25:14.289511991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:25:14.293648488Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T05:25:14.449059864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44028"
level=info ts=2026-04-21T05:25:14.683252015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:25:14.687389981Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:25:16.352866341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:25:16.356494002Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:25:17.221202948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:25:17.224693493Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:25:17.523521917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:25:17.867663611Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:25:18.189802177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:25:20.24543933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:25:20.249276479Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:25:22.223015816Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:25:23.189249451Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:25:23.189256583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:23.189272712Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:25:23.189280258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T05:25:25.250068743Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:35292"
level=info ts=2026-04-21T05:25:25.27080171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:25:25.275625422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:25:25.275650896Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:25:25.275679237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:25:25.27940803Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:25:25.70078635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:25:25.704344668Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:25:25.845778434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:25:25.851059546Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:25:26.077814138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:25:26.081584823Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:25:26.578503788Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T05:25:26.57852772Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-21T05:25:27.726934253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:25:27.733648235Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:25:28.181900389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:25:28.185629129Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:25:28.189742366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:25:29.711718439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:25:29.715447875Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:25:30.942376044Z caller=checkpoint.go:378 msg="writing series" size="16 MB"
level=debug ts=2026-04-21T05:25:30.94240656Z caller=checkpoint.go:390 msg="flushing series" totalSize="16 MB" series=1
level=debug ts=2026-04-21T05:25:31.127234566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:25:31.735348988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:25:31.739167054Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:25:33.189483407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:25:34.86852473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:25:34.872710802Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:25:35.288027734Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
ts=2026-04-21T05:25:37.348345942Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:25:38.189681356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:25:39.366401194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:25:39.369987674Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:25:39.642635288Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=info ts=2026-04-21T05:25:40.05174933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:25:40.055276037Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:25:42.100969053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:25:42.106849825Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:25:43.189971134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:43.24515935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:25:43.996766247Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
ts=2026-04-21T05:25:44.451868663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41008"
level=info ts=2026-04-21T05:25:47.912340837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:25:47.918803426Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T05:25:48.189752759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:25:48.352249944Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=info ts=2026-04-21T05:25:50.851765224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:25:50.85592317Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:25:51.058627999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:25:51.061883983Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:25:52.706447094Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-04-21T05:25:53.173227693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:25:53.173235627Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:25:53.189557724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:25:53.189590943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:25:53.189601214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:25:53.189622586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:25:55.133803984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:25:55.138077237Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:25:57.061603739Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T05:25:58.189460146Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:25:58.751006919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:25:58.755510228Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:26:01.26469949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:26:01.268124877Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:26:01.416733451Z caller=checkpoint.go:378 msg="writing series" size="6.2 kB"
level=info ts=2026-04-21T05:26:02.785550183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:26:02.789027111Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-04-21T05:26:03.190096421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:03.661990257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:26:03.665436006Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:26:03.682504373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:26:03.686064077Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:26:05.771396012Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
ts=2026-04-21T05:26:07.350645586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:26:08.085181838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:26:08.091292168Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:26:08.18978959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:26:10.126521984Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=debug ts=2026-04-21T05:26:10.126549513Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-21T05:26:12.776702982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:26:12.779921068Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:26:13.074759621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:26:13.078591524Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-21T05:26:13.189869437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:26:13.997881994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:26:14.015027081Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:26:14.481440915Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
ts=2026-04-21T05:26:15.957029074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:56144"
level=info ts=2026-04-21T05:26:17.08964792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:26:17.092910952Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:26:17.501162893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:26:17.50453742Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T05:26:18.189678286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:26:18.835738951Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-21T05:26:19.178896424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:26:19.185182348Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:26:21.013935643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:26:21.017592066Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:26:21.31490458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:26:21.318833657Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:26:22.876765257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:26:22.88046387Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:26:23.19010794Z caller=checkpoint.go:390 msg="flushing series" totalSize="117 kB" series=2
level=warn ts=2026-04-21T05:26:23.19012648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:23.190152287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:26:23.190170741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:26:23.190178988Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:26:23.195912655Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-21T05:26:23.198010868Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007975987s
ts=2026-04-21T05:26:25.256327294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:56040"
level=info ts=2026-04-21T05:26:27.327640031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:26:27.33293397Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:26:28.190121759Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:31.760716701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:26:31.768317608Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T05:26:33.189655725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:37.245266546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:26:37.249636737Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T05:26:37.354025334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:26:38.190039498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:42.134535535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:26:42.138617739Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:26:42.402662842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:26:42.406607315Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:26:43.190165638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:43.911955513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:26:43.915685721Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:26:46.109921819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:26:46.113508832Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:26:46.687097784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:26:46.693854727Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:26:47.939011898Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:26:48.18933597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:50.261193335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:26:50.264931334Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:26:53.172577855Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:26:53.173636696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:26:53.189963025Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T05:26:53.190011117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:26:53.190004684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:26:53.190030044Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:26:53.190040401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:26:53.190141501Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=debug ts=2026-04-21T05:26:53.190640576Z caller=checkpoint.go:378 msg="writing series" size="527 kB"
level=debug ts=2026-04-21T05:26:55.867286621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:26:56.005705725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:26:56.009486569Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:26:57.545603839Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=info ts=2026-04-21T05:26:58.069345693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:26:58.073182104Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:26:58.190183888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:26:59.518485949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:26:59.521686031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:26:59.521915155Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:26:59.52482931Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:27:01.901395451Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T05:27:01.901420288Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-04-21T05:27:01.98119766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:27:01.9862636Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:27:02.501674488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:27:02.506623488Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:27:02.506648297Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:27:02.506678757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:27:02.511684125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:27:03.18994319Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:04.387398488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:27:04.390634462Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:27:05.002015571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:27:05.005568502Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:27:06.2556435Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T05:27:07.355804118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:27:08.189641308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:08.750134161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:27:08.753717304Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:27:10.6106129Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-04-21T05:27:11.533019743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:27:11.536703447Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:27:12.957718084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:27:12.96114267Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T05:27:13.189313079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:27:14.965415289Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=info ts=2026-04-21T05:27:15.09787512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:27:15.102433585Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:27:15.524775109Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:27:15.946765115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:27:15.950351228Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T05:27:18.189599737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:27:19.320143271Z caller=checkpoint.go:378 msg="writing series" size="170 kB"
level=info ts=2026-04-21T05:27:20.169644464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:27:20.173376938Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:27:21.41006984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:27:21.415803901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:27:23.189713801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:27:23.189738049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:27:23.189746696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:27:23.18974577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:23.492982885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:27:23.496633033Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:27:23.675152394Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:27:26.350472025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:27:26.354228469Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:27:27.757367859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:27:27.765772841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T05:27:28.029520993Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-04-21T05:27:28.189374394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:30.435510519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:27:30.440729985Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:27:32.384190294Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T05:27:33.189525525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:35.793530793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:27:35.799044439Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:27:35.799064465Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:27:35.799087143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:27:35.802515875Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:27:36.739415277Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-04-21T05:27:36.887904934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:27:36.89182771Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-04-21T05:27:37.357255384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:27:38.18939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:27:41.094003337Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=info ts=2026-04-21T05:27:41.558417868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:27:41.561966494Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:27:43.189440272Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:43.31278402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:27:43.316360888Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:27:44.233345105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:27:44.236522442Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:27:44.261659428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:27:44.29578194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:27:44.299274252Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
ts=2026-04-21T05:27:44.460203355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47256"
level=debug ts=2026-04-21T05:27:45.448601226Z caller=checkpoint.go:378 msg="writing series" size="258 kB"
level=debug ts=2026-04-21T05:27:45.853391874Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:27:46.878256606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:27:46.882759418Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:27:48.189913218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:27:48.377669061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:27:48.380973889Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:27:48.504245233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:27:48.507510613Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:27:49.804303951Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=debug ts=2026-04-21T05:27:49.804329255Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-04-21T05:27:51.023629975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:27:51.028623909Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:27:51.706333796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:27:51.709710732Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:27:51.795085628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:27:51.798433915Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:27:53.172699083Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:27:53.173755884Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:27:53.190068836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:27:53.190084465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:27:53.190092458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:27:53.190105911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:27:54.158495034Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=warn ts=2026-04-21T05:27:58.189284944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:27:58.513699624Z caller=checkpoint.go:378 msg="writing series" size="629 kB"
level=info ts=2026-04-21T05:27:59.557730728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:27:59.561938872Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:28:01.804334301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:28:01.808785145Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:28:02.868867982Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T05:28:03.189785693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:03.255920103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:28:03.259679737Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:28:05.704666211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:28:05.708241157Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:28:07.223611742Z caller=checkpoint.go:378 msg="writing series" size="821 B"
ts=2026-04-21T05:28:07.358058376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:28:08.001764588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:28:08.009034547Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T05:28:08.190124803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:09.586430379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:28:09.590178663Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:28:10.399373499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:28:10.402989045Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:28:10.841023037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:28:10.844811359Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:28:11.577899965Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-21T05:28:12.842954506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:28:12.84703261Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:28:13.189750918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:28:14.463264107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47020"
level=debug ts=2026-04-21T05:28:14.71425177Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:28:15.933001256Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=info ts=2026-04-21T05:28:17.311047052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:28:17.314962597Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:28:18.190128021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:28:20.287344431Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T05:28:23.189471351Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:28:23.189497805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:28:23.189508252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:28:23.189506044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:23.454912715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:28:23.4587505Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:28:24.642380074Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=info ts=2026-04-21T05:28:25.528123266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="405 B" avg_comp="405 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:28:25.532513883Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:28:28.19024081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:28.21732625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:28:28.221262446Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-04-21T05:28:28.221287083Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:28:28.221312455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:28:28.224251512Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:28:28.614273713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:28:28.617609888Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:28:28.617629482Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:28:28.61765186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:28:28.620484689Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:28:28.998312072Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-21T05:28:28.998334859Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=9
level=info ts=2026-04-21T05:28:32.057204356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:28:32.063982116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:28:33.189847482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:28:33.352419755Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=info ts=2026-04-21T05:28:33.864404296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:28:33.868172503Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:28:33.86819268Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:28:33.868231424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:28:33.871445766Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-04-21T05:28:37.359660849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=debug ts=2026-04-21T05:28:37.707269555Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T05:28:38.189711338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:38.584655997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:28:38.58796276Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:28:40.281962582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:28:40.285185859Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:28:40.799701005Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:28:40.801278507Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T05:28:41.488514676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:28:41.491869661Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-04-21T05:28:41.491894575Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:28:41.491919521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:28:41.494671533Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:28:41.633245371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:28:41.638474258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:28:41.652535636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:28:41.656071869Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:28:42.062322576Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-21T05:28:43.189470851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:45.364415895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:28:45.367935873Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:28:46.416332134Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=info ts=2026-04-21T05:28:47.376468361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:28:47.37974886Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:28:48.189426311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:28:49.161562247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:28:49.164863596Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:28:49.419435183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:28:49.627992738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:28:49.631318289Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:28:49.631341947Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:28:49.631365949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:28:49.634334427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:28:50.77183227Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=info ts=2026-04-21T05:28:51.271050228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:28:51.278580563Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:28:52.665259063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:28:52.66849229Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:28:53.165017125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:28:53.168926985Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:28:53.172763065Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:28:53.173814543Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:28:53.190126115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:28:53.190133747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:28:53.190151466Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:28:53.19016701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:28:54.109055192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:28:54.112885908Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:28:55.127156994Z caller=checkpoint.go:378 msg="writing series" size="581 kB"
level=debug ts=2026-04-21T05:28:55.127181781Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-04-21T05:28:55.871319819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:28:55.875455224Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:28:56.231710371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:28:56.235076454Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:28:58.189803425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:28:59.48175474Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-04-21T05:28:59.962908801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:28:59.966663651Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:28:59.966680405Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:28:59.966699459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:28:59.969484552Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:29:02.719980804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:29:02.724281137Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-21T05:29:03.189794108Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:29:03.836273899Z caller=checkpoint.go:378 msg="writing series" size="5.5 kB"
ts=2026-04-21T05:29:07.361462091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:29:08.189975026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:29:08.191367903Z caller=checkpoint.go:378 msg="writing series" size="542 kB"
level=info ts=2026-04-21T05:29:08.404010947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:29:08.408058262Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:29:09.341834989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:29:09.345292897Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:29:12.545926782Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-04-21T05:29:12.939866048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:29:12.943693701Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:29:12.943713138Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:12.943733406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:29:12.946705329Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T05:29:13.189955045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:29:13.522132768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:29:13.525925071Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:29:13.525945303Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:13.525968181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:29:13.528674956Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:29:14.510599025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:29:14.513740292Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:29:16.14913697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:29:16.152615452Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:29:16.900271851Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=info ts=2026-04-21T05:29:18.10927787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:29:18.114359656Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:29:18.189350597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:29:18.443938243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:29:19.4598109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:29:19.463447429Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:29:21.25568458Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:29:22.139259556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:29:22.142523357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:29:23.157597489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:29:23.160849536Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:29:23.18920229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:29:23.189251266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:29:23.189262372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:29:23.189279683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:29:25.267662876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:47782"
level=debug ts=2026-04-21T05:29:25.610101053Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:29:25.703345574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:29:25.706960663Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:29:26.485516457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:29:26.488840024Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:29:27.333113756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:29:27.336458278Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:29:28.093836742Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:29:28.18958202Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:29:29.964935586Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T05:29:30.865676611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:29:30.869640297Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-21T05:29:33.190069971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:29:34.319760832Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-04-21T05:29:35.785780803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:29:35.790635342Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:29:35.790660115Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:35.790686275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:29:35.794047633Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:29:36.093859421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:29:36.097563197Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:29:36.3088455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:29:36.312200862Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-04-21T05:29:36.31223978Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:36.31226605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:29:36.315178532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-04-21T05:29:37.364452154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:29:38.189866342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:29:38.287190907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:29:38.290804663Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:29:38.674691641Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T05:29:38.866235438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:29:38.87034796Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:29:43.029552788Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-21T05:29:43.190172944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:29:44.353363036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:29:44.358865351Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:29:44.763002789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:29:46.129464131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:29:46.132783427Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T05:29:46.132806117Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:46.132829189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:29:46.136202063Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:29:47.384148913Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-21T05:29:48.18997235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:29:48.477018567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:29:48.483546813Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:29:48.483571136Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:48.483602116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.3 MB" total_uncomp="121 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:29:48.490506102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:29:49.166418093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:29:49.169746876Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:29:49.96178033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:29:49.965081245Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:29:50.096532598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:29:50.100168393Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:29:50.100187262Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:50.100224548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:29:50.102995498Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:29:51.739017232Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-04-21T05:29:53.173067284Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:29:53.173304802Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:29:53.189626654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:29:53.189646845Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:29:53.18965488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:29:53.18965641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:29:53.283077444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:29:53.286772379Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:29:54.834355674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:29:54.837853127Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:29:54.837874144Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:54.83789896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:29:54.840704903Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:29:55.423498912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:29:55.427612124Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:29:55.427642517Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:29:55.427670538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:29:55.430762766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:29:56.09396583Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=debug ts=2026-04-21T05:29:56.09398854Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=14
level=info ts=2026-04-21T05:29:56.912821804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:29:56.916316862Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:29:58.190146195Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:30:00.449297442Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-21T05:30:01.530742782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:30:01.534938479Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:30:01.801399513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:30:01.805067216Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:30:03.190025144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:30:04.80397246Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T05:30:04.968551449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:30:04.975128792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:30:05.226801627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:30:05.230093597Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:30:06.560009399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:30:06.564809687Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:30:07.107917516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:30:07.111437527Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:30:07.25088888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:30:07.254314223Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-04-21T05:30:07.367009886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:30:08.190190183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:30:09.158896243Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=warn ts=2026-04-21T05:30:13.189776598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:30:13.51391145Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:30:14.077971219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:30:14.081559501Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:30:14.081579476Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:30:14.081601695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:30:14.084541501Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:30:15.305140078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:30:15.309916361Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:30:15.309939032Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:30:15.309965501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:30:15.312836715Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:30:16.077322482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:30:16.083034872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:30:17.758655166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:30:17.86824551Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:30:18.189408444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:30:19.339884436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:30:19.343506908Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:30:21.155506302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:30:21.158880215Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:30:22.223245805Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:30:23.190244783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:30:23.190275172Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:30:23.190284923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:30:23.190313491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:30:25.272102838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:34802"
level=debug ts=2026-04-21T05:30:26.578579469Z caller=checkpoint.go:378 msg="writing series" size="969 kB"
level=debug ts=2026-04-21T05:30:26.578603569Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-21T05:30:27.01032051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:30:27.013705103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:30:27.706648041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:30:27.71006641Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:30:27.71008615Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:30:27.710105849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:30:27.712835401Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:30:27.867500469Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-21T05:30:28.189454614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:30:28.501685603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:30:28.50558975Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:30:28.648580679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:30:28.651902321Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:30:29.392402778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:30:29.395896222Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:30:29.427967536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:30:29.431315456Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-21T05:30:29.899815109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:30:29.903343589Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:30:29.99772051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:30:30.000896807Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:30:30.630638503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:30:30.634374012Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:30:30.950764114Z caller=checkpoint.go:378 msg="writing series" size="17 MB"
level=debug ts=2026-04-21T05:30:30.950787817Z caller=checkpoint.go:390 msg="flushing series" totalSize="17 MB" series=1
level=info ts=2026-04-21T05:30:31.196262404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:30:31.199680444Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:30:31.221731853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:30:31.225023377Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T05:30:33.18942037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:30:35.108787702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:30:35.112631819Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:30:35.2875077Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
level=info ts=2026-04-21T05:30:35.766898457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:30:35.770817537Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T05:30:37.369267044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:30:38.190275136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:30:39.642045174Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
level=info ts=2026-04-21T05:30:42.134267724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:30:42.13787819Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:30:42.719526316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:30:42.726642675Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:30:42.928918387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:30:42.932841114Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-21T05:30:43.189301586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:30:43.464390435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:30:43.468067599Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:30:43.996991124Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
ts=2026-04-21T05:30:44.476326997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41726"
level=info ts=2026-04-21T05:30:46.618498321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:30:46.623935028Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:30:48.189336136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:30:48.351747161Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=info ts=2026-04-21T05:30:51.076668496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:30:51.080379969Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:30:52.489970574Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:30:52.706558439Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-04-21T05:30:53.173045448Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:30:53.17330497Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=debug ts=2026-04-21T05:30:53.17457741Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-21T05:30:53.174618187Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974165 n=1
level=debug ts=2026-04-21T05:30:53.174660082Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974165 n=1
level=info ts=2026-04-21T05:30:53.189662587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:30:53.189687669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:30:53.189696968Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:30:53.189686673Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:30:55.275477479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:45964"
level=info ts=2026-04-21T05:30:56.364795876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:30:56.370801539Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:30:57.062042885Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-21T05:30:58.189530517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:31:01.416518249Z caller=checkpoint.go:378 msg="writing series" size="6.2 kB"
level=warn ts=2026-04-21T05:31:03.189651467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:04.756038566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:31:04.759928852Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:31:04.882613573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:31:04.886541416Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:31:05.771408554Z caller=checkpoint.go:378 msg="writing series" size="265 kB"
ts=2026-04-21T05:31:07.372120089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:31:08.157526531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:31:08.161159871Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:31:08.189867304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:08.64353879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:31:08.647360137Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:31:10.12692832Z caller=checkpoint.go:378 msg="writing series" size="115 kB"
level=info ts=2026-04-21T05:31:13.048260239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:31:13.05218462Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-04-21T05:31:13.13650722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:31:13.139796747Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T05:31:13.189201585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:13.575555718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:31:13.578811962Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:31:14.481100361Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-21T05:31:16.968241403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:31:16.972052051Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-21T05:31:18.189255921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:31:18.836074236Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-04-21T05:31:18.836103655Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-21T05:31:18.887265531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:31:18.890745188Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:31:23.190066576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:23.190075674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:31:23.190098686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:31:23.190107431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:31:23.19649602Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-04-21T05:31:23.198914131Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008593182s
level=info ts=2026-04-21T05:31:23.30552716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:31:23.308947086Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:31:24.021826124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:31:24.028872119Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:31:24.485228781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:31:24.488510949Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:31:24.809395265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:31:24.812608751Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:31:25.177797178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:31:25.18156168Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:31:25.986725527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:31:25.989956094Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:31:26.41906176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:31:26.421281719Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:31:26.886920804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:31:27.641331535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:31:27.645192971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:31:28.189566803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:31:28.290835565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:31:31.292117507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:31:31.295864615Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:31:33.158425415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:31:33.162006702Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-21T05:31:33.189460816Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:33.96144003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:31:33.965200407Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
ts=2026-04-21T05:31:37.375079733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=warn ts=2026-04-21T05:31:38.189709895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:39.882256757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:31:39.887959092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:31:43.189404608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T05:31:45.990831055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:37920"
level=info ts=2026-04-21T05:31:47.585015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:31:47.590910212Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-21T05:31:48.189462081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:31:52.735608207Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:31:53.173056987Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:31:53.173909974Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:31:53.189143642Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T05:31:53.18917575Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:31:53.189200975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:31:53.189228972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:31:53.189260347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:53.189338523Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=debug ts=2026-04-21T05:31:53.189654356Z caller=checkpoint.go:378 msg="writing series" size="352 kB"
level=info ts=2026-04-21T05:31:56.198679466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:31:56.20443508Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:31:57.494544049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:31:57.498506721Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:31:57.544593611Z caller=checkpoint.go:378 msg="writing series" size="5.2 kB"
level=warn ts=2026-04-21T05:31:58.189828337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:31:59.812546559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:31:59.816504445Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:32:00.206874246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:32:00.210704371Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:32:01.89990909Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T05:32:01.899931108Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=3
level=warn ts=2026-04-21T05:32:03.189969197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:04.316009453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:32:04.31959171Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:32:04.786035638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:32:04.78930273Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:32:06.255055938Z caller=checkpoint.go:378 msg="writing series" size="654 B"
ts=2026-04-21T05:32:07.37711159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=info ts=2026-04-21T05:32:07.685262462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:32:07.69214214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T05:32:08.189707425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:32:10.609003195Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-04-21T05:32:10.8466212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:32:10.850544304Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:32:11.376883782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:32:11.38027159Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:32:13.189247083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:14.901711101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:32:14.905836575Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:32:14.911882775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:32:14.914771327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:32:14.963999891Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-21T05:32:16.758711588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:32:16.762192603Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:32:17.267023446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:32:17.270742074Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:32:17.822176949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:32:17.828184249Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-04-21T05:32:18.190077818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:18.756615011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:32:18.760339022Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:32:19.319447279Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=info ts=2026-04-21T05:32:19.62928798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:32:19.632606417Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:32:19.912410061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:32:19.916246204Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T05:32:20.498987349Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:32:23.189894837Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:32:23.18991917Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-21T05:32:23.189908078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:23.189927449Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:32:23.674327056Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=info ts=2026-04-21T05:32:25.504802177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:32:25.508515493Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:32:25.508536088Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:32:25.508562687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:32:25.511603709Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:32:26.546255845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:32:26.55015577Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:32:26.644023406Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:32:28.028650766Z caller=checkpoint.go:378 msg="writing series" size="6.8 kB"
level=warn ts=2026-04-21T05:32:28.190059332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:29.857325088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:32:29.862304648Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:32:32.063557855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:32:32.067271307Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:32:32.383191241Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-21T05:32:33.189989418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:32:36.738885381Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
ts=2026-04-21T05:32:37.379456025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:32:37.532171368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:32:37.535931722Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:32:38.189613852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:39.709332404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:32:39.713570884Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:32:41.092983549Z caller=checkpoint.go:378 msg="writing series" size="233 kB"
level=warn ts=2026-04-21T05:32:43.190016267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:43.775764515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:32:43.779971909Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-04-21T05:32:44.484463138Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38938"
level=debug ts=2026-04-21T05:32:44.898796253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T05:32:45.448554874Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
ts=2026-04-21T05:32:45.995009992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.80:40728"
level=warn ts=2026-04-21T05:32:48.189412513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:32:49.700963094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:32:49.706872711Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:32:49.803496412Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=info ts=2026-04-21T05:32:51.325549392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:32:51.32995967Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:32:53.17309735Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T05:32:53.173523938Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:32:53.189810448Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:32:53.189815151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:32:53.189832791Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:32:53.189843556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T05:32:53.545385302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:32:53.549231077Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:32:54.157880461Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=debug ts=2026-04-21T05:32:54.157902607Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-04-21T05:32:54.313421199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:32:54.316902766Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-21T05:32:55.282717854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:37002"
level=info ts=2026-04-21T05:32:55.618162287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:32:55.621992193Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-04-21T05:32:58.190154677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:32:58.512601082Z caller=checkpoint.go:378 msg="writing series" size="538 kB"
level=info ts=2026-04-21T05:32:58.552139144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:32:58.555696503Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:33:00.198484151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:33:00.202360546Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:33:01.279622404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-21T05:33:01.281494836Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-21T05:33:01.784169677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:33:01.78766382Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T05:33:02.86776286Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-21T05:33:03.189257221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:04.350593312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:33:04.357256603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:33:05.578267335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="334 kB" avg_comp="334 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:33:05.582058883Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:33:07.221810107Z caller=checkpoint.go:378 msg="writing series" size="821 B"
ts=2026-04-21T05:33:07.381712056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:33:08.18965953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:08.833071733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:33:08.836459265Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:33:09.724710506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:33:09.727980845Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T05:33:10.653203737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:33:10.656865886Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-21T05:33:11.57710014Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=info ts=2026-04-21T05:33:12.699357239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:33:12.703002897Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T05:33:13.189732329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:33:13.339118246Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:33:13.954149725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:33:13.957691269Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:33:14.236673861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:33:14.240200352Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=debug ts=2026-04-21T05:33:15.931916383Z caller=checkpoint.go:378 msg="writing series" size="9.2 kB"
level=warn ts=2026-04-21T05:33:18.189707408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:33:20.286516173Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T05:33:21.603175977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:33:21.609544699Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:33:21.609564965Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:33:21.609588269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:33:21.614657965Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:33:23.035562867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:33:23.039351205Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-04-21T05:33:23.189852716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:23.189915835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:33:23.189937175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:33:23.189945896Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:33:24.642063471Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=info ts=2026-04-21T05:33:26.769719738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:33:26.77390425Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T05:33:28.18925012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:28.398133777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:33:28.402170996Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-21T05:33:28.997318514Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-21T05:33:28.997343549Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=8
level=warn ts=2026-04-21T05:33:33.189846578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:33:33.351301905Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=info ts=2026-04-21T05:33:34.198400537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:33:34.202675657Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-04-21T05:33:37.384098955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=info ts=2026-04-21T05:33:37.504801298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:33:37.510373801Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T05:33:37.706039293Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=warn ts=2026-04-21T05:33:38.190015266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:38.940346441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="405 B" avg_comp="405 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:33:38.943853584Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T05:33:40.798506827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-21T05:33:42.060581335Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=debug ts=2026-04-21T05:33:42.434062212Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:33:42.434136968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="410 kB" avg_comp="410 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T05:33:42.438004783Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dae33b2c3%3A19dae87361b%3Ad55c5299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-21T05:33:43.190162644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:43.288387485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:33:43.291797774Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-21T05:33:44.377859423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:33:44.381331553Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-04-21T05:33:44.488815494Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39812"
level=info ts=2026-04-21T05:33:44.732891085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:33:44.736644132Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:33:46.415936532Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=warn ts=2026-04-21T05:33:48.189448834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:48.505401361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:33:48.508666092Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:33:49.044251676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:33:49.048267956Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T05:33:50.770724302Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-04-21T05:33:52.184224894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:33:52.18945859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-04-21T05:33:53.172859829Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:33:53.173922348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:33:53.189164965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:33:53.189194122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:33:53.189242344Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:33:53.18922208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:53.376400656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:33:53.379666989Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:33:53.490925109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:33:53.49438435Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:33:54.478969807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:33:54.482486018Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:33:55.125475982Z caller=checkpoint.go:378 msg="writing series" size="446 kB"
ts=2026-04-21T05:33:55.287877428Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:36802"
level=info ts=2026-04-21T05:33:56.027992643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:33:56.031462414Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T05:33:57.934682099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:33:57.938409105Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-21T05:33:58.189813125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:33:59.29439467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:33:59.301574429Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:33:59.301597038Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:33:59.301624971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:33:59.304422127Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:33:59.480147253Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-21T05:33:59.480176043Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=7
level=info ts=2026-04-21T05:34:01.357561629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:34:01.36416773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:34:02.138344434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T05:34:02.14197818Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dae33b2c3%3A19dae87361b%3Ad55c5299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T05:34:03.178758008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.7 kB" avg_comp="2.7 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:34:03.182162528Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-04-21T05:34:03.189247259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:03.835231831Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB"
level=info ts=2026-04-21T05:34:07.215808266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:34:07.220519549Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-21T05:34:07.387330104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-tr45t-0b3fc3cb 10.130.0.76:7946"
level=warn ts=2026-04-21T05:34:08.189869614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:08.190024593Z caller=checkpoint.go:378 msg="writing series" size="423 kB"
level=debug ts=2026-04-21T05:34:10.31291693Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:34:10.725977993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:34:10.730989693Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T05:34:12.045425214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:34:12.049040443Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-21T05:34:12.095250635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:34:12.098333049Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-21T05:34:12.544902192Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=warn ts=2026-04-21T05:34:13.189996334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:16.899652566Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=warn ts=2026-04-21T05:34:18.189627276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:34:19.313749884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="907 B" avg_comp="907 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:34:19.317580238Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=debug ts=2026-04-21T05:34:21.254236378Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-04-21T05:34:21.622347255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:34:21.626019859Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T05:34:22.170997935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:34:22.174734307Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-21T05:34:23.190151008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:34:23.190147391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:23.190172815Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:34:23.190181054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-21T05:34:25.608961309Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T05:34:26.557335819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:34:26.562752372Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T05:34:28.189373157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:34:29.021196244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:34:29.025030879Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T05:34:29.743705228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:34:29.747385714Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:34:29.964110375Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-21T05:34:32.128340015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:34:32.131735465Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-21T05:34:33.05933381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:34:33.06317967Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T05:34:33.189510609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:34.319013367Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=debug ts=2026-04-21T05:34:35.607155095Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T05:34:37.389250024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c993d3c7 10.128.0.80:7946"
level=warn ts=2026-04-21T05:34:38.189243061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:38.364135832Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:34:38.414088271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:34:38.417323435Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T05:34:38.674286719Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-21T05:34:41.471837131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T05:34:41.475968679Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dae33b2c3%3A19dae87361b%3Ad55c5299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-21T05:34:42.078878429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:34:42.082354252Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-21T05:34:43.02905922Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-21T05:34:43.189473887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:34:44.206255362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:34:44.209961225Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-21T05:34:44.495204812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46364"
level=debug ts=2026-04-21T05:34:47.383768001Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=warn ts=2026-04-21T05:34:48.189227221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:34:48.75832926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:34:48.762335252Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:34:49.168260389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:34:49.172288655Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:34:49.172309802Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:34:49.172330837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:34:49.175280756Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:34:51.73817717Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=info ts=2026-04-21T05:34:52.612643085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T05:34:52.616786616Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dae340e22%3A19dae344515%3A49ab7ccb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T05:34:52.642849926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="65 kB" avg_comp="65 kB" total_uncomp="574 kB" avg_uncomp="574 kB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T05:34:52.646197641Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dae33c009%3A19dae350f3d%3Acfc665c1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T05:34:52.784609252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T05:34:52.788163628Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dae340e22%3A19dae344515%3A70cfd753?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T05:34:53.172789569Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.82:11211 count=1
level=info ts=2026-04-21T05:34:53.173851428Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T05:34:53.190222686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T05:34:53.190204839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:34:53.190241604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:34:53.190251277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T05:34:55.294123265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.78:49924"
level=debug ts=2026-04-21T05:34:56.093793848Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=debug ts=2026-04-21T05:34:56.093815206Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-04-21T05:34:56.377308097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.4 MB" avg_comp="1.4 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T05:34:56.382015372Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dae33b789%3A19dae3f57ac%3Aa8ef78bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T05:34:58.189594704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:35:00.44769929Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-21T05:35:00.522906626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-04-21T05:35:00.529183314Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19dae340e22%3A19dae340e23%3A423d5108?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-04-21T05:35:00.627568314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="143 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T05:35:00.633869824Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dae33a302%3A19dae34377c%3A75edcc65?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T05:35:00.984008587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:35:00.987461926Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-04-21T05:35:02.949982306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.7 kB" avg_uncomp="5.7 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-21T05:35:02.953641231Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dae340f1a%3A19dae344515%3A630074d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-21T05:35:03.189940064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:35:04.803324438Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-21T05:35:06.042034691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-21T05:35:06.047312136Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dae340e22%3A19dae344515%3Aa42456a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-21T05:35:07.391855155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-pwll8-aa07bb56 10.130.0.78:7946"
level=debug ts=2026-04-21T05:35:07.895675875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T05:35:08.189523693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:35:08.645356086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="142 kB" avg_uncomp="142 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T05:35:08.650106519Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dae340efc%3A19dae35846d%3A1ad0da9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-21T05:35:09.158127586Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=info ts=2026-04-21T05:35:09.175331713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="889 B" avg_comp="889 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-04-21T05:35:09.179014651Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1d773d0c3eb218e7/19dae344514%3A19dae344516%3Ae47d004?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-04-21T05:35:12.898870801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:35:12.903341775Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:35:12.903363808Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:35:12.903389599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:35:12.906515338Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-21T05:35:13.189947903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:35:13.512857511Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
ts=2026-04-21T05:35:14.497929399Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37252"
level=info ts=2026-04-21T05:35:15.038687299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="423 B" avg_comp="423 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T05:35:15.042091311Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dae349d93%3A19dae349d94%3A1aff248c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T05:35:15.284936902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T05:35:15.288239922Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dae33b525%3A19dae340e24%3A65258294?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T05:35:17.867781749Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T05:35:18.189861922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:35:22.019674413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-21T05:35:22.02384774Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dae33b2c3%3A19dae87361b%3Ad55c5299?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-21T05:35:22.222557114Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-21T05:35:23.189833872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T05:35:23.189860772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T05:35:23.189868741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T05:35:23.189876031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:35:23.42524884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="335 kB" avg_comp="335 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T05:35:23.42893837Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dae33b59a%3A19dae3c2b5d%3A4c5a6c59?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T05:35:24.143512348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:35:24.146861243Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:35:24.146879903Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:35:24.146903023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T05:35:24.149659744Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dae340e22%3A19dae345744%3Ad1f64733?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T05:35:24.260182583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:35:24.264686748Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:35:24.264706648Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:35:24.447112253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T05:35:24.450505656Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dae33b2c0%3A19dae340e23%3A58694f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T05:35:24.673353462Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-21T05:35:25.664158381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:35:25.667687488Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-04-21T05:35:25.667710019Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:35:25.667732655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="5.1 kB" avg_uncomp="5.1 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-04-21T05:35:25.670748698Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ea7c78ab76400a8c/19dae340f1a%3A19dae340f1b%3A666aeef9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=debug ts=2026-04-21T05:35:26.577323631Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-21T05:35:26.577355848Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-04-21T05:35:27.984277957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="14 kB" avg_comp="14 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-21T05:35:27.988108835Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dae33b294%3A19dae3925ee%3Af0f2493e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-21T05:35:28.190141629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T05:35:29.291847865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:35:29.295173723Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-04-21T05:35:29.295192476Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T05:35:29.295239241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-04-21T05:35:29.297882954Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19dae340e22%3A19dae340e23%3A86276d63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-04-21T05:35:30.422175708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="381 kB" avg_comp="381 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-21T05:35:30.425881106Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dae33b22d%3A19dae71b27d%3A1af10f43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-21T05:35:30.948832737Z caller=checkpoint.go:378 msg="writing series" size="18 MB"
level=debug ts=2026-04-21T05:35:30.948861875Z caller=checkpoint.go:390 msg="flushing series" totalSize="18 MB" series=1
level=info ts=2026-04-21T05:35:31.064758599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-04-21T05:35:31.068976953Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19dae344514%3A19dae344515%3A7e150c80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-04-21T05:35:32.004545105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T05:35:32.347449707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="55 kB" avg_uncomp="55 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-21T05:35:32.350673454Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dae340e22%3A19dae340e24%3A4d5bdc49?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-21T05:35:33.189978944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T05:35:35.286982477Z caller=checkpoint.go:378 msg="writing series" size="280 kB"