level=info ts=2026-04-21T12:15:26.687067051Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-21T12:15:26.687090404Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-21T12:15:26.687366535Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-21T12:15:26.687966218Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-21T12:15:26.688309596Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T12:15:26.688378217Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.81 inf=eth0
level=info ts=2026-04-21T12:15:26.688858509Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-58ea1800
ts=2026-04-21T12:15:26.688930334Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-21T12:15:26.688946242Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.81 advertisePort=7946
level=debug ts=2026-04-21T12:15:26.688962414Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.81 advertisePort=7946
level=info ts=2026-04-21T12:15:26.689062808Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-04-21T12:15:26.690833873Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:15:26.692278234Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-21T12:15:26.692440678Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T12:15:26.693101897Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-21T12:15:26.693122509Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-21T12:15:26.693120741Z
level=debug ts=2026-04-21T12:15:26.693140822Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-21T12:15:26.693168221Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-21T12:15:26.693184323Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-04-21T12:15:26.693190518Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-21T12:15:26.693189621Z
level=debug ts=2026-04-21T12:15:26.693194948Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-21T12:15:26.69582408Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-21T12:15:26.695891574Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.81 inf=eth0
ts=2026-04-21T12:15:26.696096674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.82:7946"
ts=2026-04-21T12:15:26.697613136Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.76:7946: dial tcp 10.130.0.76:7946: connect: connection refused"
ts=2026-04-21T12:15:26.698215304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.74:7946"
ts=2026-04-21T12:15:26.699496471Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.79:7946: dial tcp 10.130.0.79:7946: connect: connection refused"
ts=2026-04-21T12:15:26.700019025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
ts=2026-04-21T12:15:26.701156283Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
level=info ts=2026-04-21T12:15:26.701166025Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=12.104823ms
level=info ts=2026-04-21T12:15:26.701181059Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-04-21T12:15:26.703527125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.82:7946"
ts=2026-04-21T12:15:26.704201205Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.76:7946: dial tcp 10.130.0.76:7946: connect: connection refused"
ts=2026-04-21T12:15:26.704372277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.74:7946"
ts=2026-04-21T12:15:26.704829003Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.79:7946: dial tcp 10.130.0.79:7946: connect: connection refused"
ts=2026-04-21T12:15:26.704977529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.80:7946"
ts=2026-04-21T12:15:26.705394551Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
level=info ts=2026-04-21T12:15:26.705402774Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=4.217519ms
level=info ts=2026-04-21T12:15:26.710734444Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-21T12:15:26.710772918Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-21T12:15:26.710776997Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-21T12:15:26.710797375Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-21T12:15:26.710797325Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-04-21T12:15:26.710805352Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-21T12:15:26.710888215Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-21T12:15:26.710997923Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-21T12:15:26.711098058Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-21T12:15:26.711129855Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-21T12:15:26.711140821Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-21T12:15:26.711148526Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-04-21T12:15:26.711148846Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-21T12:15:26.711153251Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-21T12:15:26.711151832Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-21T12:15:26.711134716Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-21T12:15:26.711398543Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=info ts=2026-04-21T12:15:26.711402652Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-21T12:15:26.711437686Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-21T12:15:26.711441315Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-21T12:15:26.711444094Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-21T12:15:26.711446549Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-21T12:15:26.711448859Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-21T12:15:26.711451149Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-21T12:15:26.711460328Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-21T12:15:26.85945262Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=147.992687ms errors=false
level=info ts=2026-04-21T12:15:26.859484034Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-21T12:15:28.602958953Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=1.891499361s errors=false
level=info ts=2026-04-21T12:15:28.602985206Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-21T12:15:28.603016115Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=1.891556976s
level=info ts=2026-04-21T12:15:28.603037197Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-21T12:15:28.603067063Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=8.43050635s
level=info ts=2026-04-21T12:15:28.603076781Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-21T12:15:28.603124637Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-04-21T12:15:28.603153291Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-21T12:15:28.603186728Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-21T12:15:28.603204257Z caller=loki.go:599 msg="Loki started" startup_time=1.970540967s
level=warn ts=2026-04-21T12:15:33.60402522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:15:38.603712976Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:15:42.58946963Z caller=logging.go:137 msg="GET /ready (503) 94.354µs"
level=warn ts=2026-04-21T12:15:43.604004783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:15:48.603516943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:15:52.587394927Z caller=logging.go:137 msg="GET /ready (503) 82.891µs"
level=warn ts=2026-04-21T12:15:53.604230429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:15:58.603974488Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-21T12:15:58.604017659Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-21T12:15:58.604031656Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=warn ts=2026-04-21T12:15:58.604024901Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:15:58.604040713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-21T12:16:02.844078826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:49512"
level=warn ts=2026-04-21T12:16:03.603515708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:16:08.603498045Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:16:10.606630014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=info ts=2026-04-21T12:16:10.841004332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=info ts=2026-04-21T12:16:10.841025204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:16:10.841025192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:16:10.841032073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-04-21T12:16:10.841050853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="180 kB" avg_uncomp="180 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:16:10.841072205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:16:10.841092453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="974 kB" avg_comp="974 kB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:16:10.841119336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=info ts=2026-04-21T12:16:10.841116436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="54 kB" avg_uncomp="54 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-21T12:16:10.841151141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T12:16:10.841169168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T12:16:10.841200878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:16:10.841198813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=info ts=2026-04-21T12:16:10.841220312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=info ts=2026-04-21T12:16:10.841237625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:16:10.841312563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T12:16:10.841418527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="420 kB" avg_comp="420 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T12:16:10.841679893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=debug ts=2026-04-21T12:16:10.841765007Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-21T12:16:10.847691915Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:16:10.847739303Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T12:16:10.847766963Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:16:10.847776386Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:16:10.84779541Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:16:10.847811087Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:16:10.847812756Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:16:10.847806791Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:16:10.847825344Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:16:10.847806861Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:16:10.847850556Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:16:10.84784585Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:16:10.847852943Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:16:10.847850523Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:16:10.847856957Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:16:10.850150668Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=error ts=2026-04-21T12:16:10.850150486Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:16:10.850230135Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T12:16:13.603942579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:16:18.603353005Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:16:21.771319257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:42102"
level=info ts=2026-04-21T12:16:22.000378725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:16:22.005673783Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-21T12:16:22.640696079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:16:22.644062557Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-21T12:16:23.603794751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:16:23.851888111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:16:23.855136425Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T12:16:25.044072366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:16:25.051770603Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=info ts=2026-04-21T12:16:25.889812471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:16:25.89416239Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T12:16:26.263374404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:16:26.267293258Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:16:26.404685412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T12:16:26.626384311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:16:26.629629802Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T12:16:26.692192224Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:16:26.693241425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-21T12:16:27.641061685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:49410"
level=info ts=2026-04-21T12:16:27.808910338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:16:27.811937149Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T12:16:27.987428554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:16:27.995163789Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:16:28.423223655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:16:28.426417043Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=info ts=2026-04-21T12:16:28.54077586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:16:28.543871309Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:16:28.570932699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:16:28.57784456Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T12:16:28.603211235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:16:28.603227828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:16:28.603235513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:16:28.603240559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:16:29.631215502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:16:29.635644343Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:16:29.69598783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:16:29.699489287Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T12:16:29.709574945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:16:29.713575448Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:16:30.384761119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:16:30.388717619Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T12:16:30.485766191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:16:30.488693344Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:16:30.760946084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:16:30.764148469Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T12:16:33.603931553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:16:37.868407757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T12:16:38.60345118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:16:40.608247941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=warn ts=2026-04-21T12:16:43.603728135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:16:48.381965103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:16:48.385834794Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T12:16:48.603460079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:16:48.64660517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:16:48.649640205Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=info ts=2026-04-21T12:16:48.81564256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:16:48.819724701Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:16:50.175240098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T12:16:51.758897159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:16:51.764050413Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:16:52.489161938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:16:52.49257144Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T12:16:53.603949643Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:16:53.68112322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:16:53.684716368Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:16:53.84600332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:16:53.84935502Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-21T12:16:54.051411546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:16:54.054641357Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:16:54.614254462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:16:54.617294622Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:16:58.603423304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:16:58.603459348Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:16:58.603467693Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:16:58.603471021Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:17:00.001607361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:17:00.005426382Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T12:17:00.026500104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:17:00.034979275Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=info ts=2026-04-21T12:17:00.132294737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:17:00.141203924Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:17:00.649024858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:17:00.653875465Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:17:01.073861636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:17:01.077498552Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:17:03.354031131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:17:03.359557322Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:17:03.600208632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T12:17:03.603334683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-21T12:17:03.604390913Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T12:17:04.691643145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:17:04.699192986Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T12:17:06.333786408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:17:06.338088797Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T12:17:08.603590089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:17:09.680901595Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:17:09.680983864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:17:09.689793211Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T12:17:10.143492805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-21T12:17:10.609050413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=warn ts=2026-04-21T12:17:13.60371766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:17:18.603964321Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:17:21.773223633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:46324"
level=warn ts=2026-04-21T12:17:23.604019728Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:17:26.692003447Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:17:26.693046268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-21T12:17:28.603958705Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:17:28.603951466Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:17:28.603984072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:17:28.603993739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T12:17:28.65309098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:17:28.656678899Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T12:17:28.948004348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:17:28.951975775Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:17:29.089803593Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T12:17:33.572768881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:17:33.576529209Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=warn ts=2026-04-21T12:17:33.603733866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:17:38.604016514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:17:40.610309431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=info ts=2026-04-21T12:17:41.054238947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:17:41.057646587Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T12:17:43.604162233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:17:43.789045657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:17:43.792264031Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-21T12:17:44.164515658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:17:44.169063734Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:17:44.349631762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:17:44.352925081Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:17:44.705868999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:17:44.709913223Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T12:17:45.439525059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:17:46.227397363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:17:46.23059117Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:17:47.527738962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:17:47.53109379Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:17:48.481687994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:17:48.485250842Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T12:17:48.603777303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:17:50.191686264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:17:50.195416684Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T12:17:53.434567878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:17:53.4413658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T12:17:53.604109938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:17:53.606159973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:17:53.609284413Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:17:54.296982857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:17:54.300348153Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:17:54.920075217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:17:54.92673456Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=info ts=2026-04-21T12:17:56.620232885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:17:56.624261192Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T12:17:56.742618097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:17:56.746997487Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:17:56.795139384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:17:56.79796463Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T12:17:57.737390843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:17:57.743880528Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:17:58.604015404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T12:17:58.604014564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:17:58.60404222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:17:58.60405033Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:18:03.603659888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:18:04.896103155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T12:18:04.897570233Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-21T12:18:08.603772489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:18:10.611606947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=warn ts=2026-04-21T12:18:13.604214936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:18:13.672473409Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:18:18.057955384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:18:18.06408525Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-21T12:18:18.603755142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:22.964256266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:18:22.969563236Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=warn ts=2026-04-21T12:18:23.604222753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:25.459129537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:18:25.46277931Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-04-21T12:18:26.692089381Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:18:26.693152162Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T12:18:28.604226544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T12:18:28.604234497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:18:28.604257061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:18:28.604265758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T12:18:33.007393349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:18:33.010655637Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T12:18:33.603661483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:36.052959088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:18:36.056159434Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:18:36.650991295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:18:36.654121888Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:18:37.839371801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:18:37.843628011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-21T12:18:38.60377681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:39.022940259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:18:39.029446305Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:18:39.328276236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:18:39.333107486Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:18:39.619859133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:18:39.623153655Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T12:18:40.536998482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:18:40.54022804Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-21T12:18:40.612589918Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-19ec21e9 10.130.0.82:7946"
level=debug ts=2026-04-21T12:18:40.667987148Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T12:18:40.816298566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:18:40.819829074Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:18:41.191784191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:18:41.195172447Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T12:18:41.743501564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:18:41.767539105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:18:41.772340416Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:18:41.851528541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:18:41.85478894Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:18:41.973162919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:18:41.977401479Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T12:18:43.60332809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:43.691033123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:18:43.695729687Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T12:18:48.603789316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:53.148924327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:18:53.155946369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-21T12:18:53.604195536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:18:54.364270415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:18:54.371247455Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
ts=2026-04-21T12:18:57.64690582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:46566"
level=info ts=2026-04-21T12:18:58.603964099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T12:18:58.603970773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:18:58.603994208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:18:58.604002955Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:19:03.603470577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:19:08.603784033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:08.892158099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:19:08.898316466Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:19:10.581129339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-21T12:19:10.613574683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=info ts=2026-04-21T12:19:13.49018823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:19:13.494433877Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-04-21T12:19:13.603777618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:17.315262849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:19:17.318382869Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=warn ts=2026-04-21T12:19:18.603944424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:18.756790894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:19:18.761001599Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T12:19:20.992530057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:19:20.997329741Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T12:19:21.778420455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:38844"
level=info ts=2026-04-21T12:19:22.487782202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:19:22.49240156Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T12:19:23.048039426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:19:23.051538566Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-21T12:19:23.604092804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:19:26.692141861Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:19:26.693188384Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T12:19:27.239999673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:19:27.243686192Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:19:27.812593473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:19:27.816059603Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:19:28.603797024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T12:19:28.603791967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:19:28.603820581Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:19:28.603829466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T12:19:29.035592337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:19:29.039410026Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T12:19:29.516466901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T12:19:31.292258535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:19:31.298795647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:19:33.209340442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:19:33.213468387Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T12:19:33.603523959Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:33.711321786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:19:33.714465637Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:19:33.822425325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:19:33.827222078Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:19:34.90825496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:19:34.912285484Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T12:19:35.305440733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:19:35.308846375Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-21T12:19:35.828522343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:19:35.831659243Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-21T12:19:38.603917552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:39.518391362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:19:39.525145152Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-21T12:19:40.615174721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:19:43.208628939Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T12:19:43.603738006Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-21T12:19:48.604000497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:50.999357754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:19:51.009825909Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=warn ts=2026-04-21T12:19:53.60390738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:19:58.290102769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:19:58.295355217Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T12:19:58.604220146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T12:19:58.604230334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:19:58.604250804Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:19:58.604258547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T12:19:59.69150705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:19:59.695072907Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-04-21T12:20:02.854672514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:58336"
level=warn ts=2026-04-21T12:20:03.603511729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:20:07.462137483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:20:07.466047184Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:20:08.453302267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:20:08.456762922Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=warn ts=2026-04-21T12:20:08.604232088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:20:08.913069226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:20:08.918022425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:20:09.375848607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:20:09.378999803Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:20:10.266510709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:20:10.269865252Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T12:20:10.61617868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=info ts=2026-04-21T12:20:11.157729551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:20:11.161175259Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:20:12.922241683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:20:12.929276605Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-21T12:20:13.604266378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:20:14.71428586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:20:14.719086327Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:20:15.00979325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:20:15.013614796Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T12:20:18.045518598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:20:18.524789005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:20:18.528507988Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T12:20:18.603718175Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:20:20.175182902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:20:20.178998219Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:20:21.189785316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:20:21.193400755Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-04-21T12:20:21.780912407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:54630"
level=warn ts=2026-04-21T12:20:23.604255871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:20:24.525280345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:20:24.529354166Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T12:20:25.628932208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:20:25.635393523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T12:20:26.128275338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:20:26.131318499Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T12:20:26.692312196Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:20:26.693374586Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-21T12:20:27.650694852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:38412"
level=info ts=2026-04-21T12:20:28.6039204Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-21T12:20:28.603977657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-21T12:20:28.603983426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:20:28.603999289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:20:28.604010091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-21T12:20:28.604117252Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=debug ts=2026-04-21T12:20:28.60461909Z caller=checkpoint.go:378 msg="writing series" size="692 kB"
level=info ts=2026-04-21T12:20:31.516497925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:20:31.520717128Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-21T12:20:31.8980099Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=warn ts=2026-04-21T12:20:33.603851369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:20:35.193121222Z caller=checkpoint.go:378 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-21T12:20:35.193144437Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.5 MB" series=3
level=info ts=2026-04-21T12:20:35.917435736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:20:35.925565612Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=debug ts=2026-04-21T12:20:38.482972953Z caller=checkpoint.go:378 msg="writing series" size="475 kB"
level=warn ts=2026-04-21T12:20:38.60341284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:20:40.617289957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:20:41.77545316Z caller=checkpoint.go:378 msg="writing series" size="146 kB"
level=debug ts=2026-04-21T12:20:42.080893269Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-21T12:20:43.454707416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:20:43.458399997Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-04-21T12:20:43.603943367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:20:45.068494083Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-21T12:20:48.361449821Z caller=checkpoint.go:378 msg="writing series" size="219 kB"
level=warn ts=2026-04-21T12:20:48.604171826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:20:51.653698757Z caller=checkpoint.go:378 msg="writing series" size="488 kB"
level=debug ts=2026-04-21T12:20:51.653721438Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-21T12:20:52.907854753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:20:52.913895551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-21T12:20:53.603789287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:20:53.630939931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T12:20:53.631039341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-21T12:20:53.638030891Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T12:20:54.946124926Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=info ts=2026-04-21T12:20:56.616105163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:20:56.620712783Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-21T12:20:58.204196261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:20:58.207210378Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T12:20:58.23946991Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=info ts=2026-04-21T12:20:58.603662365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:20:58.60369251Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:20:58.603700332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:20:58.603719551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:00.112779664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:21:00.118014714Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:21:00.463795602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:21:00.467481208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:21:00.624144991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:21:00.629200091Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-21T12:21:01.531393438Z caller=checkpoint.go:378 msg="writing series" size="208 kB"
level=info ts=2026-04-21T12:21:01.749788408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:21:01.753403154Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
ts=2026-04-21T12:21:02.85731216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59496"
level=info ts=2026-04-21T12:21:03.516562143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:21:03.519878912Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-21T12:21:03.604128203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:03.782708356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:21:03.786160081Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-21T12:21:04.824117804Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-04-21T12:21:05.289399179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:21:05.293173395Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:21:05.604059345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:21:05.60826627Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T12:21:08.117091692Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-21T12:21:08.603470538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:21:10.618301223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:21:11.410516878Z caller=checkpoint.go:378 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-21T12:21:11.410546491Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=6
level=info ts=2026-04-21T12:21:12.059915087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:21:12.064229335Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T12:21:13.603960048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:13.905162634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:21:13.911645557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T12:21:14.702421228Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-04-21T12:21:15.661035428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:21:15.665525653Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T12:21:16.819703384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:21:16.822949287Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-21T12:21:17.995152755Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=warn ts=2026-04-21T12:21:18.603403586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:21:19.401905818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:21:19.549266132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:21:19.557361871Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=debug ts=2026-04-21T12:21:21.289024624Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-21T12:21:21.289053185Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-04-21T12:21:23.604081254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:24.013426895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:21:24.017220572Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T12:21:24.580303538Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=debug ts=2026-04-21T12:21:26.692212321Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:21:26.693279953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-21T12:21:27.45679197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:21:27.46122897Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-04-21T12:21:27.874284273Z caller=checkpoint.go:378 msg="writing series" size="739 kB"
level=info ts=2026-04-21T12:21:28.603311278Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:21:28.603338435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:21:28.603349171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:21:28.603380657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:21:31.165663569Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=debug ts=2026-04-21T12:21:31.165688659Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-21T12:21:33.603984418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:34.241162086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:21:34.24962209Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-21T12:21:34.458207267Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=debug ts=2026-04-21T12:21:37.750881641Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-21T12:21:38.603388025Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:39.961174632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:21:39.966514931Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:21:40.211133377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:21:40.214346643Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-21T12:21:40.619714508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-nwjzv-ab708c63 10.130.0.74:7946"
level=debug ts=2026-04-21T12:21:41.043684226Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=warn ts=2026-04-21T12:21:43.603445714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:44.085796217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:21:44.089233877Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T12:21:44.337609588Z caller=checkpoint.go:378 msg="writing series" size="659 kB"
level=debug ts=2026-04-21T12:21:44.607878167Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-21T12:21:46.92343771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:21:46.926776314Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-21T12:21:47.629181518Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=warn ts=2026-04-21T12:21:48.603539059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:21:50.922126045Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-04-21T12:21:51.537512194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:21:51.541672197Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-21T12:21:51.784915889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:46306"
level=debug ts=2026-04-21T12:21:51.882649762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-21T12:21:53.603668426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:53.815583018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:21:53.819580063Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:21:54.214760661Z caller=checkpoint.go:378 msg="writing series" size="590 kB"
level=debug ts=2026-04-21T12:21:54.21479024Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=7
level=info ts=2026-04-21T12:21:54.374861845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:21:54.378420598Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:21:55.315212461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:21:55.319479363Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T12:21:57.507291814Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=info ts=2026-04-21T12:21:57.541414794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:21:57.545057982Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-21T12:21:57.654345832Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36988"
level=info ts=2026-04-21T12:21:58.379066118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:21:58.38413922Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:21:58.603870896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:21:58.603902672Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:21:58.603913533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:21:58.603907417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:21:59.556265852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:21:59.55973596Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=info ts=2026-04-21T12:22:00.517280286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:22:00.521124135Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-21T12:22:00.799932965Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=info ts=2026-04-21T12:22:02.513224695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:22:02.51685389Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-21T12:22:03.604104577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:04.09243928Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=info ts=2026-04-21T12:22:04.836486923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:22:04.842725005Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-21T12:22:05.071923066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:22:05.080245263Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=debug ts=2026-04-21T12:22:07.386812929Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-21T12:22:07.38683629Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=4
level=info ts=2026-04-21T12:22:07.686303563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="5.9 kB" avg_comp="5.9 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:22:07.690878619Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:22:07.690908296Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:07.690939874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:22:07.694460798Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-21T12:22:08.276132663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:22:08.280377531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-21T12:22:08.603386382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-21T12:22:10.621028209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:22:10.679971834Z caller=checkpoint.go:378 msg="writing series" size="2.9 MB"
level=debug ts=2026-04-21T12:22:10.679987689Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=1
level=debug ts=2026-04-21T12:22:10.735889523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T12:22:13.603376333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:13.971309573Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=info ts=2026-04-21T12:22:16.5187847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:22:16.522614941Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T12:22:17.263489294Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-21T12:22:18.60342706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:20.556299864Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
ts=2026-04-21T12:22:21.786280248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.74:50338"
level=warn ts=2026-04-21T12:22:23.604020249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:23.849309136Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-04-21T12:22:24.493275884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:22:24.500376079Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:22:24.500396739Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:24.500432469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:22:24.507731791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-21T12:22:24.844564484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:22:24.847873054Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-04-21T12:22:26.692264941Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.76:11211 count=1
level=info ts=2026-04-21T12:22:26.69329599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-21T12:22:27.141101741Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
level=info ts=2026-04-21T12:22:27.445785945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:22:27.449217539Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:22:27.449237443Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:27.449258651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:22:27.452120667Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-21T12:22:27.656288103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.82:36810"
level=info ts=2026-04-21T12:22:28.6034561Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:22:28.603488705Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:22:28.603502388Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:22:28.60349997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:30.433769176Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-21T12:22:32.021773467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:22:32.026343933Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:22:32.026364753Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:32.026388541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:22:32.030339092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-21T12:22:32.86159205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.80:59800"
level=warn ts=2026-04-21T12:22:33.604154938Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:33.726329385Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-21T12:22:34.804711389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-21T12:22:37.019234337Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-21T12:22:37.448462394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:22:37.454192115Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:22:37.454213382Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:37.454234901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:22:37.457764372Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-21T12:22:37.547120262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:22:37.550181385Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:22:37.550197709Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:37.754857276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:22:37.758205267Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-21T12:22:38.603528677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:22:38.66067649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:22:38.664333975Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-21T12:22:38.798276065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:22:38.801461889Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:22:38.801479113Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:38.801499941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:22:38.804218344Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:22:39.897080131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:22:39.90091018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:22:39.900928453Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:39.900948761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:22:39.904136317Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-21T12:22:40.312397892Z caller=checkpoint.go:378 msg="writing series" size="335 kB"
level=info ts=2026-04-21T12:22:40.380433985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:22:40.387825472Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-21T12:22:40.622656384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=info ts=2026-04-21T12:22:41.591536311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:22:41.594864155Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:22:41.594883017Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:41.594904492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:22:41.597781664Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=warn ts=2026-04-21T12:22:43.603492896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-21T12:22:43.607963097Z caller=checkpoint.go:378 msg="writing series" size="3.4 MB"
level=debug ts=2026-04-21T12:22:43.607979894Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.2 MB" series=10
level=info ts=2026-04-21T12:22:44.750899632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:22:44.75566449Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:22:44.755683547Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:44.755705128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:22:44.760278719Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:22:45.260694247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:22:45.264042235Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-21T12:22:45.264063351Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:45.264084594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:22:45.266797881Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:22:45.48738646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:22:45.492233836Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-21T12:22:45.526798771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:22:45.530021442Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:22:45.530039424Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:45.564160029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:22:45.567271281Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:22:46.255928154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:22:46.259816166Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:22:46.259836251Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:46.259858688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:22:46.263225137Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:22:46.897812753Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-21T12:22:48.603813151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:22:49.604679573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:22:49.612175089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:22:49.612196371Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:49.612223939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:22:49.619275968Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T12:22:50.190038726Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T12:22:51.730760232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-21T12:22:51.734336625Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dafc0c4d2%3A19dafc20524%3Ac0cd6c8b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-21T12:22:51.96191108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:22:51.966488759Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:22:51.96650986Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:51.966534844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:22:51.970504203Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T12:22:52.282254539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:22:52.286513971Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-21T12:22:53.483125296Z caller=checkpoint.go:378 msg="writing series" size="636 kB"
level=warn ts=2026-04-21T12:22:53.603644839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:22:53.920859258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="180 kB" avg_uncomp="180 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-21T12:22:53.924680395Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dafb2b6d5%3A19dafbb2b3c%3A51e24e66?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-21T12:22:53.941723345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:22:53.94591903Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T12:22:55.375922162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="54 kB" avg_uncomp="54 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-04-21T12:22:55.37933839Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19dafb1c49e%3A19dafba73d5%3A880a7b5d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-04-21T12:22:55.621073796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-21T12:22:55.624476723Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dafb2845b%3A19dafb2845c%3Adfbf5da7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-21T12:22:56.376242842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:22:56.37982982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:22:56.379851153Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:56.379872555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:22:56.382901839Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-21T12:22:56.775247435Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-21T12:22:58.004690436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-21T12:22:58.008170585Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dafb1c438%3A19dafb2b71d%3A38717c1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-21T12:22:58.484566036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:22:58.492649486Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=error ts=2026-04-21T12:22:58.49266889Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:22:58.49269049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:22:58.50014Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=info ts=2026-04-21T12:22:58.603413478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-21T12:22:58.603436459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-21T12:22:58.603446414Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-21T12:22:58.603452955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:22:59.619991449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=254f2d3c6973a5ea immediate=false num_chunks=1 total_comp="662 B" avg_comp="662 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ywdw\"}"
level=error ts=2026-04-21T12:22:59.623408153Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/254f2d3c6973a5ea/19dafc04bcf%3A19dafc06b2d%3Aa9a62dc1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ywdw\"}"
level=debug ts=2026-04-21T12:23:00.068870891Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=debug ts=2026-04-21T12:23:00.068891161Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-21T12:23:00.967527106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="9.8 MB" avg_uncomp="9.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-21T12:23:00.972044536Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dafb1c51a%3A19dafd837ec%3Ab5d1aba8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-21T12:23:02.342680698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="734 kB" avg_comp="734 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-21T12:23:02.347046396Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dafb1c402%3A19dafeb47a4%3A4c3c473f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-21T12:23:03.062513975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="421 kB" avg_comp="421 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-21T12:23:03.066391378Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dafb1c5be%3A19dafec1ead%3Ab6a03b2c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-21T12:23:03.361417381Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-04-21T12:23:03.604180902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:23:03.609337993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-21T12:23:03.612582609Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dafb28480%3A19dafb2b71d%3A8a06da99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-21T12:23:04.567127162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="113 MB" avg_uncomp="23 MB" full=5 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-21T12:23:04.574448197Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dafb1c79e%3A19dafb29d3b%3A16d4ded5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-21T12:23:06.530593333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-21T12:23:06.532375971Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-21T12:23:06.653819813Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=debug ts=2026-04-21T12:23:07.441890065Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-21T12:23:08.604201941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:23:09.157166165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-21T12:23:09.163726378Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dafb29d39%3A19dafb2b71d%3A79b0de54?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-21T12:23:09.946808159Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-21T12:23:10.623639802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-2h5lb-d50d75d1 10.130.0.80:7946"
level=debug ts=2026-04-21T12:23:13.239084416Z caller=checkpoint.go:378 msg="writing series" size="587 kB"
level=info ts=2026-04-21T12:23:13.476370983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c6e3db7eb8bd7c77 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-kage\"}"
level=error ts=2026-04-21T12:23:13.484539104Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c6e3db7eb8bd7c77/19dafc116db%3A19dafccd49f%3A9b1f96cd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-kage\"}"
level=warn ts=2026-04-21T12:23:13.604011946Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:23:14.57847789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-21T12:23:14.582050242Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dafb2b6d5%3A19dafb2b71d%3A87acba39?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-21T12:23:14.99636651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:23:15.000078878Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:23:15.000099975Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-21T12:23:15.000122791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-21T12:23:15.003132933Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dafb2845b%3A19dafb2b74d%3A7c5cc8a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-21T12:23:16.531892149Z caller=checkpoint.go:378 msg="writing series" size="59 kB"
level=info ts=2026-04-21T12:23:18.405061777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-21T12:23:18.40956018Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dafb1c744%3A19dafcbdfff%3A518e15d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-21T12:23:18.60415978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-21T12:23:18.831117973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-21T12:23:18.83594503Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dafb1c49f%3A19dafc0f7b0%3A703c6c05?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-21T12:23:18.883054926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.7 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-21T12:23:18.887990225Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dafb1c74b%3A19dafbfe411%3Aaef18718?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-21T12:23:19.824350802Z caller=checkpoint.go:378 msg="writing series" size="207 kB"
level=debug ts=2026-04-21T12:23:19.824381441Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-04-21T12:23:20.246418604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-21T12:23:20.254230685Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dafb1c447%3A19dafc11647%3A1bd6700a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"