level=info ts=2026-05-13T12:44:11.351068454Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T12:44:11.351088735Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-13T12:44:11.351494054Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-13T12:44:11.35273708Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T12:44:11.353361363Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T12:44:11.35346952Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.58 inf=eth0
level=info ts=2026-05-13T12:44:11.354276649Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-15e5a2c0
ts=2026-05-13T12:44:11.354355379Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T12:44:11.354370665Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.58 advertisePort=7946
level=debug ts=2026-05-13T12:44:11.354390718Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.58 advertisePort=7946
level=info ts=2026-05-13T12:44:11.354536732Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-13T12:44:11.358129439Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T12:44:11.358159822Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
ts=2026-05-13T12:44:11.359064582Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T12:44:11.359087003Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:44:11.35910011Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=4.564914ms
level=error ts=2026-05-13T12:44:11.359112699Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T12:44:11.35912234Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-13T12:44:11.359408796Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-13T12:44:11.359589285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T12:44:11.359903827Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-13T12:44:11.359919921Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-13T12:44:11.359917983Z
level=debug ts=2026-05-13T12:44:11.359941199Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-13T12:44:11.359961519Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-13T12:44:11.359979591Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-13T12:44:11.360098322Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-13T12:44:11.359985388Z
level=debug ts=2026-05-13T12:44:11.360126475Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-13T12:44:11.362588918Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T12:44:11.362665775Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.58 inf=eth0
ts=2026-05-13T12:44:11.363223356Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:11.363241185Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T12:44:11.377133852Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-13T12:44:11.377137534Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-13T12:44:11.377156336Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-13T12:44:11.377162657Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-13T12:44:11.377193588Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-13T12:44:11.377216595Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-13T12:44:11.377214345Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-13T12:44:11.37722026Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-13T12:44:11.377290214Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-13T12:44:11.377284834Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-13T12:44:11.377375134Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-13T12:44:11.377381297Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-13T12:44:11.377379374Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-13T12:44:11.37738526Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-05-13T12:44:11.377390238Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-13T12:44:11.377397879Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-13T12:44:11.377427343Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-13T12:44:11.37745348Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-13T12:44:11.377458123Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-13T12:44:11.37746114Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-13T12:44:11.377570955Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-13T12:44:11.377575537Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-13T12:44:11.377578657Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-13T12:44:11.377581346Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-05-13T12:44:11.377583977Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-13T12:44:11.377605866Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-13T12:44:11.377613296Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T12:44:11.377649935Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.764µs success=true
level=info ts=2026-05-13T12:44:11.37766297Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-13T12:44:11.378037863Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=427.445µs errors=false
level=info ts=2026-05-13T12:44:11.378052696Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-13T12:44:11.378188205Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=583.549µs errors=false
level=info ts=2026-05-13T12:44:11.37819674Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-13T12:44:11.378202452Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=597.175µs
level=info ts=2026-05-13T12:44:11.37822157Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-13T12:44:11.378239957Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T12:44:11.378245026Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=21.131741206s
level=info ts=2026-05-13T12:44:11.378258354Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2026-05-13T12:44:11.378333033Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-13T12:44:11.37833116Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-13T12:44:11.378338092Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-05-13T12:44:11.378348918Z caller=loki.go:615 msg="Loki started" startup_time=106.608197ms
level=debug ts=2026-05-13T12:44:11.378430275Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-13T12:44:13.088317347Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:13.088339993Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:44:16.378460253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:44:16.644048129Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:16.644071386Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:44:21.378697633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:44:22.684874028Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T12:44:22.684895261Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:44:26.379367993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:44:26.919872334Z caller=logging.go:137 msg="GET /ready (503) 82.549µs"
level=warn ts=2026-05-13T12:44:31.378781129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:44:36.084552124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.55:7946"
ts=2026-05-13T12:44:36.085934827Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.56:7946: dial tcp 10.128.0.56:7946: connect: connection refused"
ts=2026-05-13T12:44:36.086593933Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.57:7946"
ts=2026-05-13T12:44:36.087913666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.54:7946"
level=info ts=2026-05-13T12:44:36.088331089Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.729202265s
level=warn ts=2026-05-13T12:44:36.378556884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:44:36.919612302Z caller=logging.go:137 msg="GET /ready (503) 93.747µs"
level=info ts=2026-05-13T12:44:41.378426426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T12:44:41.378463137Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-13T12:44:41.378475022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:44:41.378512397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:44:43.308599307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35446"
ts=2026-05-13T12:44:43.946329795Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T12:44:46.378921617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:44:51.379166452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:44:56.378867375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:44:59.980539682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:45:01.379400137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:45:06.379345406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:45:11.359364661Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:45:11.360401421Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:45:11.361741207Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T12:45:11.361807263Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976306 n=1
level=debug ts=2026-05-13T12:45:11.361863328Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976306 n=1
level=info ts=2026-05-13T12:45:11.378796353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:45:11.378819195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:45:11.378827116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:45:11.378855142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:45:13.786841781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T12:45:13.949762511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T12:45:16.06805776Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42578"
level=warn ts=2026-05-13T12:45:16.379007898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:45:21.378448164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:45:26.110720479Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:45:26.378535922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:45:31.379053994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:45:36.379257104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:45:38.180549338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T12:45:41.379301887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T12:45:41.37930553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:45:41.379331098Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:45:41.37933943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T12:45:43.951337482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T12:45:46.378933768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:45:51.378700827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:45:56.378952139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:45:57.257487682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:46:01.378789076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:46:01.447035321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:46:01.447077192Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T12:46:01.45588657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T12:46:06.37877202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:46:11.359647274Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:46:11.359922782Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:46:11.379296147Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:46:11.379333717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T12:46:11.379325626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:46:11.379345992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T12:46:13.952770301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T12:46:14.707133065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:46:14.714633431Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T12:46:16.378595949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:46:21.379151324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:46:22.153911185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:46:26.379252201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:46:29.129473122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:46:31.378660454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:46:36.378866588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:46:41.091223855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:46:41.098411837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T12:46:41.379155508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:46:41.379201755Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:46:41.379209861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:46:41.379198952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:46:43.954117001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T12:46:46.379112379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:46:51.379215966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:46:56.378996909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:46:57.918732794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T12:46:57.920433101Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T12:46:59.811787952Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:47:01.378920562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:47:06.379122711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:47:11.360071616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:47:11.360518348Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:47:11.37884536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:47:11.378862289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:47:11.378870363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:47:11.378921317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:47:13.958577301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T12:47:16.378730609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:47:21.378966483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:47:26.378977031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:47:31.379028717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:47:32.320846809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:47:32.326968825Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:47:33.429283822Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:47:34.7568917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:47:36.37915498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:47:41.378325312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:47:41.378360207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:47:41.378369218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:47:41.378401507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:47:43.960937734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T12:47:46.074164903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35018"
level=warn ts=2026-05-13T12:47:46.379199207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:47:51.379310143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:47:56.378427926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:48:01.378969515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:48:06.379048152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:48:07.783206739Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T12:48:11.360430787Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:48:11.360825434Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:48:11.379197851Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:48:11.379218188Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:48:11.379229097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:48:11.379228537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:48:13.96245846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T12:48:16.379232505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:48:19.129306262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:48:19.13623403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T12:48:21.378541066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:48:26.379472542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:48:30.263062839Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:58352"
level=warn ts=2026-05-13T12:48:31.378790044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:48:32.953317117Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:48:36.378729282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:48:41.378371434Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:48:41.378394377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:48:41.378402087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:48:41.378401814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:48:43.319558089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56240"
ts=2026-05-13T12:48:43.963356854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T12:48:46.378571949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:48:47.305255226Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T12:48:51.379067948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:48:56.378380777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:49:01.379064317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:05.054471059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:49:06.379295941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:49:11.359853989Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:49:11.36035378Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:49:11.37870022Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T12:49:11.378734144Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T12:49:11.378729455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:11.378757222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:49:11.378765421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T12:49:11.378829603Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-13T12:49:11.378851878Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T12:49:11.37887458Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.916µs success=true
level=debug ts=2026-05-13T12:49:11.379083554Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T12:49:13.964714136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:49:15.734267965Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
ts=2026-05-13T12:49:16.078226566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47184"
level=warn ts=2026-05-13T12:49:16.37912894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:49:17.286444692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:49:17.293572565Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:49:20.089333582Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-13T12:49:21.379382707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:24.444411631Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T12:49:26.378997997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:28.79932091Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T12:49:31.379275351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:33.15407972Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-13T12:49:36.379266621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:36.495642107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:49:37.508314621Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T12:49:41.378409683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:49:41.378427534Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:49:41.378450327Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:49:41.378461332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:49:41.863419943Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
ts=2026-05-13T12:49:43.322164929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40810"
ts=2026-05-13T12:49:43.966197438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T12:49:46.079331864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48846"
level=debug ts=2026-05-13T12:49:46.218147597Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T12:49:46.378528611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:50.573595497Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T12:49:51.379282463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:51.560228124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T12:49:54.927708512Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T12:49:56.378610935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:49:59.282341133Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-13T12:50:00.266962364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48382"
level=warn ts=2026-05-13T12:50:01.379255838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:03.624448627Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:50:03.637598339Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=warn ts=2026-05-13T12:50:06.379266462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:07.993041187Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-13T12:50:09.703875972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:50:09.710830884Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T12:50:11.359714813Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:50:11.359919882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:50:11.379288523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T12:50:11.379290551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:11.379305988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:50:11.379313568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:50:12.347745161Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T12:50:13.323042151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36402"
ts=2026-05-13T12:50:13.967746522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T12:50:16.379347929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:16.70232297Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-13T12:50:21.056871429Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-13T12:50:21.056894851Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=17
level=warn ts=2026-05-13T12:50:21.379370569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:25.411914825Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T12:50:26.378865662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:29.766626253Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
ts=2026-05-13T12:50:30.268718944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:43144"
level=warn ts=2026-05-13T12:50:31.378957162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:34.121504822Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T12:50:34.559063706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:50:36.378790694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:38.476958891Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=warn ts=2026-05-13T12:50:41.378701091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:50:41.378711532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:50:41.378732129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:50:41.378739332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:50:42.831507105Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-13T12:50:43.968917155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
ts=2026-05-13T12:50:46.082205083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:34546"
level=warn ts=2026-05-13T12:50:46.379000822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:47.186262545Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T12:50:51.378686014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:50:51.541323325Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=info ts=2026-05-13T12:50:51.914298469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:50:51.922661517Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:50:55.896209803Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-13T12:50:56.378473327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:00.250565068Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-13T12:51:00.250595911Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-05-13T12:51:00.26998659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:55022"
level=warn ts=2026-05-13T12:51:01.379087298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:03.099684819Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:51:03.148597851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T12:51:04.605650751Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T12:51:06.378570635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:08.959914962Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-13T12:51:11.359435738Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:51:11.36048524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T12:51:11.378863906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:51:11.378881841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:51:11.378898482Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:51:11.378905899Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:51:13.315497748Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-13T12:51:13.325704426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46698"
ts=2026-05-13T12:51:13.970434892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:51:16.083652926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40664"
level=warn ts=2026-05-13T12:51:16.378672697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:17.669982529Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-13T12:51:21.378709557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:22.025150706Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-13T12:51:26.378409698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:26.379546103Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=debug ts=2026-05-13T12:51:30.734605288Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-13T12:51:31.379188564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:35.089619763Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-13T12:51:36.378915556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:38.016600502Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:51:39.443657654Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T12:51:41.378940818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:51:41.378962324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T12:51:41.3789524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:51:41.378969833Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T12:51:43.326885935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:36172"
level=debug ts=2026-05-13T12:51:43.798577441Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-13T12:51:43.971288784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T12:51:46.378755978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:51:47.104840113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:51:47.112326705Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:51:48.154430623Z caller=checkpoint.go:372 msg="writing series" size="390 kB"
level=debug ts=2026-05-13T12:51:48.154453489Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-13T12:51:51.379282085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:52.508311712Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T12:51:56.379302804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:51:56.863293003Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=debug ts=2026-05-13T12:51:58.228061026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T12:51:58.229590291Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T12:52:00.271922564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56516"
level=debug ts=2026-05-13T12:52:01.217835974Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T12:52:01.379233679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:05.57271579Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-13T12:52:06.379434578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:09.928534953Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T12:52:09.928556455Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-13T12:52:11.359757671Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:52:11.360114924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=warn ts=2026-05-13T12:52:11.378506187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:52:11.378516427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:52:11.378540469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:52:11.378548216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:52:13.173752215Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T12:52:13.97232157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T12:52:14.282750518Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
ts=2026-05-13T12:52:16.085819357Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59486"
level=warn ts=2026-05-13T12:52:16.378723714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:18.637285532Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-13T12:52:21.379084652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:22.992954388Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-13T12:52:26.37866463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:27.350088608Z caller=checkpoint.go:372 msg="writing series" size="5.1 MB"
level=debug ts=2026-05-13T12:52:27.350110471Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=4
ts=2026-05-13T12:52:30.272936445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54994"
level=warn ts=2026-05-13T12:52:31.378877322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:31.70275202Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=info ts=2026-05-13T12:52:33.685543923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:52:33.693034399Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:52:33.693055918Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T12:52:33.693076679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:52:33.699880343Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:52:36.05747439Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-13T12:52:36.378827846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:40.412043943Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-13T12:52:40.86841069Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T12:52:41.37913201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T12:52:41.379136714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:41.379154812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:52:41.379163157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T12:52:43.973287154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:52:44.766469314Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
ts=2026-05-13T12:52:46.086549502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51394"
level=info ts=2026-05-13T12:52:46.176210359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:52:46.184704112Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T12:52:46.378373071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:49.121756066Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T12:52:51.378685723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:52.287830665Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T12:52:53.476113708Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-13T12:52:56.379120174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:52:57.830730028Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-13T12:53:00.274443609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33890"
level=warn ts=2026-05-13T12:53:01.378633544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:02.186251603Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-13T12:53:06.379424268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:06.54111437Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=debug ts=2026-05-13T12:53:07.583264202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:53:10.896194425Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=info ts=2026-05-13T12:53:11.360004679Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:53:11.360021417Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=warn ts=2026-05-13T12:53:11.378402394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:53:11.378422765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:53:11.378438505Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:53:11.378446183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T12:53:13.330624891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:44094"
ts=2026-05-13T12:53:13.974446689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:53:15.250414044Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-13T12:53:16.378896091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:19.605415739Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-13T12:53:21.378704896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:23.960667764Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-13T12:53:24.043800897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:53:24.050807397Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T12:53:26.37941147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:28.315021057Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-13T12:53:31.379366757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:32.670336Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-13T12:53:36.379086633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:37.024893127Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=debug ts=2026-05-13T12:53:37.079069817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T12:53:41.379351511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:53:41.379348936Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=16
level=debug ts=2026-05-13T12:53:41.379373224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:53:41.379381493Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:53:41.379383173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:53:41.38536704Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-13T12:53:41.388412531Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009373538s
ts=2026-05-13T12:53:43.975495302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T12:53:46.37897492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:53:50.298514468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T12:53:51.378778026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:53:56.379009294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:54:00.276994381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56526"
level=warn ts=2026-05-13T12:54:01.378426415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:54:06.379058316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:09.632776346Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T12:54:11.360342297Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:54:11.360740758Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:54:11.379202231Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T12:54:11.379207715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:54:11.379227611Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:54:11.379236897Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:54:11.379244734Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T12:54:11.379317045Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-13T12:54:11.379325058Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T12:54:11.379341949Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.749µs success=true
level=debug ts=2026-05-13T12:54:11.379486831Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T12:54:13.976457972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:54:15.735087961Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=warn ts=2026-05-13T12:54:16.378378678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:20.090194926Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-13T12:54:21.378705423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:54:23.102450093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:54:23.11082741Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:54:24.444644415Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T12:54:26.379190338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:28.799591068Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T12:54:30.278019889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54752"
level=warn ts=2026-05-13T12:54:31.379373643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:33.154619511Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-13T12:54:34.54066175Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:54:36.378951712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:37.509214261Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T12:54:41.379287569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:54:41.379324017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:54:41.379334786Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:54:41.379321072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:41.863396114Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-13T12:54:43.334721049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:46930"
ts=2026-05-13T12:54:43.978249814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T12:54:46.219133204Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-13T12:54:46.379004065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:48.535750177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T12:54:50.573242782Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T12:54:51.379057983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:54.927975087Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T12:54:56.378509364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:54:59.283111179Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T12:55:01.37898524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:03.638699718Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-13T12:55:06.379136357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:07.113772133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:55:07.992456066Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-13T12:55:11.360018177Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:55:11.360727722Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:55:11.379166599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:55:11.379192429Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:55:11.379200241Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:55:11.379232837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:12.347727216Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T12:55:13.979843158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T12:55:16.379202095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:16.702454371Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=debug ts=2026-05-13T12:55:21.05754114Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=debug ts=2026-05-13T12:55:21.05756273Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=17
level=warn ts=2026-05-13T12:55:21.379024319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:55:22.169115113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:55:22.180196331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:55:25.412689482Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T12:55:26.37902156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:29.767381175Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T12:55:31.37910014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:32.013870288Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:55:34.12252263Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T12:55:36.37912839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:38.477252093Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=warn ts=2026-05-13T12:55:41.379034459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:55:41.379044774Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:55:41.379070327Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:55:41.379081301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:55:41.666782505Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T12:55:42.83183136Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
ts=2026-05-13T12:55:43.338087078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42832"
ts=2026-05-13T12:55:43.981609193Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T12:55:46.095227465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:50536"
level=warn ts=2026-05-13T12:55:46.378416777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:47.186573492Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T12:55:51.37879208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:55:51.541076554Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=debug ts=2026-05-13T12:55:55.896621181Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-13T12:55:55.896640873Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-13T12:55:56.379413644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:00.25333554Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-13T12:56:01.379290011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:01.438512984Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:56:04.606088615Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T12:56:06.379232053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:08.961062978Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T12:56:11.360034263Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:56:11.36044712Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:56:11.378926893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:56:11.378946187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:56:11.378954455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:56:11.37901892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:13.316075946Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
ts=2026-05-13T12:56:13.339519703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:45756"
ts=2026-05-13T12:56:13.982490461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
ts=2026-05-13T12:56:16.097155567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54552"
level=warn ts=2026-05-13T12:56:16.378886929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:17.670553211Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-13T12:56:21.378470603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:21.395368274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T12:56:21.397284211Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T12:56:22.007846021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:56:22.016339296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:56:22.025497391Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=warn ts=2026-05-13T12:56:26.379104471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:26.380286353Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=debug ts=2026-05-13T12:56:30.73497242Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-13T12:56:31.378796889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:32.781493503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T12:56:32.781548829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T12:56:32.79068402Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T12:56:35.08952987Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-13T12:56:36.378946022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:39.444756579Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-05-13T12:56:39.444791128Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T12:56:41.379273193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:56:41.379301127Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:56:41.379321675Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:56:41.379321493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:43.799883574Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
ts=2026-05-13T12:56:43.983591415Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T12:56:46.098376061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42494"
level=warn ts=2026-05-13T12:56:46.379311264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:48.15808535Z caller=checkpoint.go:372 msg="writing series" size="5.2 MB"
level=debug ts=2026-05-13T12:56:48.15812322Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.5 MB" series=2
level=info ts=2026-05-13T12:56:49.141945225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T12:56:49.150092201Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T12:56:51.378984277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:52.508863592Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T12:56:56.379415224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:56:56.863767549Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-13T12:56:56.970094654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T12:57:00.909512489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:57:01.218318131Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T12:57:01.378589235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:05.573248945Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-13T12:57:06.379280325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:09.929726Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T12:57:09.92975417Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-13T12:57:11.360159436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:57:11.360626337Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:57:11.379126213Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:57:11.379148422Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:57:11.379156733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:57:11.3791607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:57:13.793611053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:57:13.802772324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T12:57:13.984501526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:57:14.282830026Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
ts=2026-05-13T12:57:16.100104599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44920"
level=warn ts=2026-05-13T12:57:16.378512469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:57:16.778042144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T12:57:16.785329624Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T12:57:18.638550245Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-13T12:57:21.378978304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:22.993473474Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=warn ts=2026-05-13T12:57:26.378854115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:27.350965899Z caller=checkpoint.go:372 msg="writing series" size="5.2 MB"
level=debug ts=2026-05-13T12:57:27.350990257Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.7 MB" series=4
ts=2026-05-13T12:57:30.28705941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:51922"
level=warn ts=2026-05-13T12:57:31.379241542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:31.702158748Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-13T12:57:31.915794787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:57:36.057599117Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=warn ts=2026-05-13T12:57:36.378555007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:40.411924237Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-13T12:57:41.378577342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:57:41.378601495Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:57:41.378610012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:57:41.378611992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:57:43.343856887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:58226"
ts=2026-05-13T12:57:43.985648308Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T12:57:44.767641223Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
ts=2026-05-13T12:57:46.101321158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57136"
level=warn ts=2026-05-13T12:57:46.378524096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:49.122515413Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T12:57:51.379274538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:53.476917332Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-13T12:57:56.379429439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:57:57.50770431Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:57:57.831683583Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-13T12:57:58.242533843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T12:57:59.140565612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:57:59.148867395Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T12:58:01.188953717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T12:58:01.195651524Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T12:58:01.378751016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:02.186777037Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-13T12:58:06.378835542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:06.541445597Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=debug ts=2026-05-13T12:58:10.896575715Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-13T12:58:11.359707518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:58:11.360104765Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:58:11.37869833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:58:11.378733735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:58:11.378745008Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T12:58:11.378744894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T12:58:13.345041513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56094"
ts=2026-05-13T12:58:13.98689251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T12:58:15.251351315Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-13T12:58:16.103237977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56922"
level=warn ts=2026-05-13T12:58:16.379520752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:19.60619519Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-13T12:58:21.379203032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:23.960822514Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-13T12:58:26.379308801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:28.31552451Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T12:58:28.315550194Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-13T12:58:31.378568892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:32.670697734Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-13T12:58:32.696944839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:58:36.378821934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:37.025168916Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-05-13T12:58:41.379357918Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T12:58:41.37936292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:58:41.379381962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:58:41.379391438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:58:41.380452109Z caller=checkpoint.go:384 msg="flushing series" totalSize="288 kB" series=2
level=info ts=2026-05-13T12:58:41.385451016Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-13T12:58:41.386913589Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007456983s
ts=2026-05-13T12:58:43.346394588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37290"
ts=2026-05-13T12:58:43.988379827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T12:58:45.886356058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T12:58:45.897690347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T12:58:46.37908683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:58:46.549976437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:58:46.557508005Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:58:49.311884735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T12:58:51.378707568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:58:56.378899563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:01.291000281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T12:59:01.379274874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T12:59:06.378620437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:59:11.359641437Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T12:59:11.359943239Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T12:59:11.378375227Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:59:11.378394736Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T12:59:11.37839153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:59:11.37840308Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T12:59:11.37840418Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T12:59:11.378525202Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-13T12:59:11.378535181Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T12:59:11.378552094Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.116µs success=true
level=debug ts=2026-05-13T12:59:11.378726787Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T12:59:13.989883886Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T12:59:14.886213544Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-13T12:59:16.379196894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:18.392743747Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-13T12:59:21.379217176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:21.898508379Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T12:59:25.405005112Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-13T12:59:26.379021995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:59:26.994106979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:59:27.0009914Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:59:27.001014433Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T12:59:27.001036479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:59:27.006834742Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T12:59:28.354676535Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T12:59:28.911720801Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T12:59:31.378735237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:32.4187419Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T12:59:35.924546318Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T12:59:36.379427548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:37.677810295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T12:59:39.431277373Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T12:59:40.982587907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T12:59:40.994942249Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T12:59:41.38071488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T12:59:41.380820447Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T12:59:41.380834629Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T12:59:41.380845391Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T12:59:42.937863423Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
ts=2026-05-13T12:59:43.991373952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T12:59:44.955365141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T12:59:44.969292249Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T12:59:46.378600904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:46.446978867Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-13T12:59:49.951050756Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T12:59:51.378509711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:53.457353114Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T12:59:56.379196789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T12:59:56.963421423Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-13T13:00:00.297876529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56700"
level=debug ts=2026-05-13T13:00:00.470651964Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-13T13:00:00.470679767Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-05-13T13:00:01.37870196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:01.503060468Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:00:03.976850562Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T13:00:06.378982758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:07.483727257Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-13T13:00:10.990189344Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:00:11.360017927Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:00:11.3607955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=debug ts=2026-05-13T13:00:11.360977167Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T13:00:11.361035621Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976307 n=1
level=debug ts=2026-05-13T13:00:11.361088074Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976307 n=1
level=info ts=2026-05-13T13:00:11.379288017Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:00:11.379316051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:00:11.379327317Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:00:11.379313025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:00:13.993137179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:00:14.496263166Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-13T13:00:16.378642278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:00:16.670815942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:00:16.678224119Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:00:18.002519779Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T13:00:21.378710506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:21.509376588Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-13T13:00:25.015515936Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T13:00:26.378640779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:28.522680341Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-13T13:00:29.435004927Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:00:30.597009703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:00:30.605817822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:00:31.379151584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:31.75828714Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:00:31.7601959Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:00:32.028242572Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T13:00:35.535201579Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T13:00:36.379145483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:39.04158149Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-13T13:00:41.378519253Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:00:41.378527335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:41.378546685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:00:41.378554726Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:00:42.548059472Z caller=checkpoint.go:372 msg="writing series" size="603 kB"
level=debug ts=2026-05-13T13:00:42.548087336Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=12
ts=2026-05-13T13:00:43.353043677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47726"
ts=2026-05-13T13:00:43.995002757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:00:46.054451177Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-13T13:00:46.378997762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:49.561263057Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:00:51.37869041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:53.067392557Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-13T13:00:56.132870197Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:00:56.379375688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:00:56.573947366Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:01:00.080950606Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-13T13:01:01.378666508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:03.587513768Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-13T13:01:06.379070948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:07.093092577Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-13T13:01:10.600570679Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-13T13:01:11.360198207Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:01:11.36063202Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:01:11.379195355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:01:11.379213217Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:01:11.379222015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:01:11.379305314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:01:13.354489713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35594"
ts=2026-05-13T13:01:13.997102451Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:01:14.107035695Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=warn ts=2026-05-13T13:01:16.379427617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:01:16.526805947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:01:16.538061283Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:01:17.613376711Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-13T13:01:21.119060797Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-13T13:01:21.379079487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:23.14808819Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:01:24.626557513Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-13T13:01:25.76574263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:01:25.773624323Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:01:26.378991245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:28.132312605Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-05-13T13:01:28.132335195Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=debug ts=2026-05-13T13:01:30.227041859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:01:30.302711157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53596"
level=warn ts=2026-05-13T13:01:31.379118736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:31.639076625Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-05-13T13:01:35.145240173Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=warn ts=2026-05-13T13:01:36.37862073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:38.651646573Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-13T13:01:41.378369651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:01:41.378393468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:01:41.378401743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:01:41.378413456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:42.158555395Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
ts=2026-05-13T13:01:43.355552674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53194"
ts=2026-05-13T13:01:43.998867476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:01:45.665158688Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
ts=2026-05-13T13:01:46.114032143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60118"
level=warn ts=2026-05-13T13:01:46.378898697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:49.171238602Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-13T13:01:50.884016338Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:01:51.378851044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:01:52.688387479Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T13:01:52.68841232Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=debug ts=2026-05-13T13:01:56.184797365Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T13:01:56.379213635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:01:56.808037083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:01:56.815945588Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:01:59.690698944Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-13T13:02:01.379271767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:03.197024886Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-13T13:02:06.378606616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:06.703956192Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-13T13:02:10.210801219Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T13:02:11.360532653Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:02:11.361076014Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:02:11.378693082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:02:11.378717989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:02:11.378711552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:02:11.378727465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:02:13.718292588Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T13:02:13.718317381Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
ts=2026-05-13T13:02:14.000011609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:02:14.895559692Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:02:16.379057977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:17.223740606Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-13T13:02:17.280932526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:02:17.288012032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:02:18.178371732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:02:20.730519663Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-13T13:02:21.379342962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:24.236296138Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-13T13:02:26.37894631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:27.742729336Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
ts=2026-05-13T13:02:30.306551263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49080"
level=debug ts=2026-05-13T13:02:31.253056727Z caller=checkpoint.go:372 msg="writing series" size="6.1 MB"
level=debug ts=2026-05-13T13:02:31.25307939Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.7 MB" series=5
level=warn ts=2026-05-13T13:02:31.378942968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:34.756481817Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-13T13:02:36.379361885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:38.263358889Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=warn ts=2026-05-13T13:02:41.378763569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:02:41.378819197Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:02:41.378840402Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:02:41.378848091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:02:41.76852331Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
ts=2026-05-13T13:02:44.001453218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:02:44.680205447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:02:45.275808945Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-13T13:02:46.116648975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35850"
level=warn ts=2026-05-13T13:02:46.37856435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:48.782198969Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-13T13:02:51.378847683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:02:52.288305751Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-13T13:02:55.79476264Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-13T13:02:56.379296313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:02:56.699769398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:02:56.707008359Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:02:57.426581775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:02:59.301537053Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-13T13:03:01.379248853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:02.807788146Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-13T13:03:05.094455588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:03:05.103421833Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:03:05.103445885Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:03:05.10346944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:03:05.110104773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:03:06.314593173Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=warn ts=2026-05-13T13:03:06.379257471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:09.820385082Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-13T13:03:11.360152964Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:03:11.360724636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:03:11.37834374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:03:11.378368194Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:03:11.378378647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:03:11.378396299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:13.327103654Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-13T13:03:13.327128546Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
ts=2026-05-13T13:03:13.360393478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:33376"
level=debug ts=2026-05-13T13:03:13.98762098Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:03:14.003106136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:03:16.379349503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:16.834252255Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-13T13:03:19.291401909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:03:19.302603583Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:03:20.340536484Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-13T13:03:21.379022191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:23.846812761Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-13T13:03:26.378875516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:27.352900824Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-13T13:03:30.309114207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41840"
level=debug ts=2026-05-13T13:03:30.859638125Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:03:31.378962678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:34.366967145Z caller=checkpoint.go:372 msg="writing series" size="390 kB"
level=warn ts=2026-05-13T13:03:36.379263272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:03:37.873070398Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-13T13:03:38.873481339Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:03:39.614458604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:03:39.623685719Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:03:41.379134437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:03:41.379148876Z caller=checkpoint.go:384 msg="flushing series" totalSize="951 kB" series=7
level=debug ts=2026-05-13T13:03:41.379162677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:03:41.379170498Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:03:41.37922431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:03:41.425123203Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-13T13:03:41.430403182Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.051718415s
level=debug ts=2026-05-13T13:03:41.947673181Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:03:44.004749947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:03:46.378947874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:03:46.646477261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:03:46.655352755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:03:51.379219168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:03:56.379232835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:04:00.310855966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42842"
level=warn ts=2026-05-13T13:04:01.378759152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:04:06.379546972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:09.958601482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:04:11.359756867Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:04:11.360740259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:04:11.378539992Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:04:11.378541535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:04:11.378567394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:04:11.378578022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:04:11.378588683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:04:11.378651781Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-13T13:04:11.378660195Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:04:11.378677507Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.086µs success=true
level=debug ts=2026-05-13T13:04:11.378834883Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T13:04:14.006396989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:04:14.88569463Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
ts=2026-05-13T13:04:16.121723136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42698"
level=warn ts=2026-05-13T13:04:16.379030003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:18.393057831Z caller=checkpoint.go:372 msg="writing series" size="319 kB"
level=warn ts=2026-05-13T13:04:21.379070179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:21.898741962Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-13T13:04:24.691751199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:04:24.701365984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:04:25.405656032Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-13T13:04:26.37903829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:27.032535095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:04:27.034286266Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:04:28.912197475Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:04:31.378513146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:32.418093453Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-13T13:04:35.45612658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="120 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:04:35.465406853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:04:35.924726478Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T13:04:36.379042904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:39.431079362Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-13T13:04:41.378724634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:04:41.378721219Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:04:41.378750278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:04:41.378761393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:04:42.938428777Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
ts=2026-05-13T13:04:44.008355412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:04:44.786652464Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:04:46.379015046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:46.444682664Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:04:46.444712271Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-05-13T13:04:49.950500023Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:04:51.379032615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:53.458208138Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T13:04:56.378917603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:04:56.964060268Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-13T13:05:00.470294393Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-13T13:05:01.379258816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:03.977041229Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T13:05:06.378438636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:07.483408707Z caller=checkpoint.go:372 msg="writing series" size="413 kB"
level=debug ts=2026-05-13T13:05:10.98925478Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:05:11.360473Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:05:11.361072844Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=warn ts=2026-05-13T13:05:11.37842143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:05:11.378427118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:05:11.378451855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:05:11.378462913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T13:05:14.009519171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:05:14.496190167Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=debug ts=2026-05-13T13:05:14.661008515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:05:16.378407682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:05:17.090714788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:05:17.099449179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:05:18.003237133Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=debug ts=2026-05-13T13:05:18.962692191Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:05:21.37890491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:21.509501787Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-13T13:05:21.509530609Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-13T13:05:24.070456422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="143 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:05:24.077207966Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:05:25.016050519Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T13:05:26.379417987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:28.522265814Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
ts=2026-05-13T13:05:30.314928109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42858"
level=warn ts=2026-05-13T13:05:31.379221784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:32.029183796Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T13:05:35.535028262Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T13:05:36.379226815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:39.042508772Z caller=checkpoint.go:372 msg="writing series" size="432 kB"
level=info ts=2026-05-13T13:05:41.378440275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:05:41.378450898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:41.378467257Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:05:41.378475723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:05:42.548876774Z caller=checkpoint.go:372 msg="writing series" size="585 kB"
level=debug ts=2026-05-13T13:05:42.548903812Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
ts=2026-05-13T13:05:44.011158687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:05:46.055324076Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T13:05:46.378528754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:49.561662375Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:05:51.378868349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:51.673761351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:05:52.41859659Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:05:53.068443452Z caller=checkpoint.go:372 msg="writing series" size="383 kB"
level=warn ts=2026-05-13T13:05:56.378750808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:05:56.57544939Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:06:00.080421981Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-13T13:06:00.316434777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:39116"
level=warn ts=2026-05-13T13:06:01.378822342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:03.587442729Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=warn ts=2026-05-13T13:06:06.378438241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:07.094736821Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T13:06:07.094758621Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=7
level=info ts=2026-05-13T13:06:07.583048086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:06:07.594198971Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:06:07.594224649Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:06:07.594252676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:06:07.600306882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:06:07.88316489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="143 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:06:07.890063921Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:06:10.59971709Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-13T13:06:11.359638775Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:06:11.36003456Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:06:11.378527759Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:06:11.37854798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:06:11.378542726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:06:11.378556825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T13:06:14.012401519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:06:14.106377101Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-13T13:06:16.378444336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:17.613111753Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-13T13:06:21.119892388Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-13T13:06:21.379226759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:24.62644992Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-13T13:06:26.379243218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:06:26.743960496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:06:26.75654561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:06:27.314292743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:06:28.076809828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:06:28.133218099Z caller=checkpoint.go:372 msg="writing series" size="717 kB"
level=debug ts=2026-05-13T13:06:28.133240437Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-13T13:06:30.318120981Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:34602"
level=warn ts=2026-05-13T13:06:31.378443998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:31.638961683Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-13T13:06:35.148497752Z caller=checkpoint.go:372 msg="writing series" size="426 kB"
level=warn ts=2026-05-13T13:06:36.378402851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:38.651630968Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T13:06:41.378820004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:06:41.378819518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:41.378856707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:06:41.378867178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:06:42.158298465Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-13T13:06:44.013785324Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:06:45.665101175Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-13T13:06:46.37854247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:49.172109547Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-13T13:06:51.379128803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:52.714664558Z caller=checkpoint.go:372 msg="writing series" size="24 MB"
level=debug ts=2026-05-13T13:06:52.714691019Z caller=checkpoint.go:384 msg="flushing series" totalSize="25 MB" series=7
level=debug ts=2026-05-13T13:06:56.184332412Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T13:06:56.379297572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:06:57.042421864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:06:59.691157384Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
ts=2026-05-13T13:07:00.319021895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54810"
level=warn ts=2026-05-13T13:07:01.378723846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:07:02.74421371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="191 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:07:02.75229656Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:07:03.197706377Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-13T13:07:06.043704107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:07:06.050397191Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:07:06.378983185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:06.703605284Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-13T13:07:10.210274566Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:07:11.359569685Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:07:11.360644956Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:07:11.379212408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:07:11.379236109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:07:11.379230352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:07:11.379245787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T13:07:13.371461554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38178"
level=debug ts=2026-05-13T13:07:13.719227299Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T13:07:13.719250116Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
ts=2026-05-13T13:07:14.014840788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T13:07:16.378590254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:17.223667829Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-13T13:07:20.729864484Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-05-13T13:07:21.378781793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:21.692995077Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:07:24.236949455Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=debug ts=2026-05-13T13:07:24.470455009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:07:26.378431464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:27.743841427Z caller=checkpoint.go:372 msg="writing series" size="580 kB"
level=debug ts=2026-05-13T13:07:27.743864467Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-13T13:07:31.25289805Z caller=checkpoint.go:372 msg="writing series" size="6.7 MB"
level=debug ts=2026-05-13T13:07:31.252925131Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.7 MB" series=1
level=warn ts=2026-05-13T13:07:31.378775489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:34.75567123Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-13T13:07:36.379376079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:38.263047368Z caller=checkpoint.go:372 msg="writing series" size="859 kB"
level=warn ts=2026-05-13T13:07:41.378807573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:07:41.378821216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:07:41.378842449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:07:41.378849741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:07:41.769321492Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-13T13:07:41.76935637Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-05-13T13:07:44.016278223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:07:45.275615892Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-13T13:07:46.379119819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:48.78151439Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-13T13:07:51.378861219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:51.763992638Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:07:52.288512683Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-13T13:07:55.795396197Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-13T13:07:56.379153126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:07:59.301710591Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-13T13:08:00.718383089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:08:00.727421984Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:08:01.378545788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:02.808251145Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-13T13:08:04.867647758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:08:04.874956961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:08:06.313965105Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=warn ts=2026-05-13T13:08:06.378971805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:09.821265369Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=debug ts=2026-05-13T13:08:11.359282901Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:08:11.360330878Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:08:11.37886403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:08:11.378868432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:11.378888707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:08:11.378899199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:08:12.888342801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:08:12.890094194Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:08:13.327377448Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
ts=2026-05-13T13:08:14.017839523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
ts=2026-05-13T13:08:16.133452007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56064"
level=warn ts=2026-05-13T13:08:16.378817108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:16.834361451Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T13:08:20.341681952Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-13T13:08:21.379063841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:23.847044435Z caller=checkpoint.go:372 msg="writing series" size="493 kB"
level=debug ts=2026-05-13T13:08:23.84707085Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-13T13:08:26.378973766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:27.352907134Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-13T13:08:27.482435345Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:08:30.322981427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:60386"
level=debug ts=2026-05-13T13:08:30.859980824Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:08:31.378935092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:34.366876736Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=warn ts=2026-05-13T13:08:36.378759006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:37.873066328Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:08:41.37852531Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:08:41.378550596Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:08:41.37855805Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:08:41.378543999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:08:41.379641945Z caller=checkpoint.go:384 msg="flushing series" totalSize="561 kB" series=4
level=info ts=2026-05-13T13:08:41.386219968Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-13T13:08:41.387214396Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008418287s
ts=2026-05-13T13:08:44.019162313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
ts=2026-05-13T13:08:46.134678975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:57078"
level=warn ts=2026-05-13T13:08:46.378620245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:08:51.379262264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:08:53.80711128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:08:53.814938082Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:08:56.379030885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:08:57.209943561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:08:57.217163731Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T13:09:00.324352166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57686"
level=debug ts=2026-05-13T13:09:01.098083123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:09:01.379006865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:09:06.379056746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:08.8924161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:09:11.359230581Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:09:11.360271007Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:09:11.378814118Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:09:11.378843601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:09:11.37885673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:11.37886667Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:09:11.378879341Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:09:11.378924595Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-13T13:09:11.378933154Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:09:11.378952415Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.235µs success=true
level=debug ts=2026-05-13T13:09:11.379130704Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T13:09:14.020886834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:09:14.841452923Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=warn ts=2026-05-13T13:09:16.378852074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:18.303084863Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-13T13:09:21.378919147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:21.76484493Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:09:25.226313348Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-05-13T13:09:25.226336845Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-13T13:09:25.961308327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:09:26.378776066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:28.687495137Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T13:09:30.325430239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57152"
level=warn ts=2026-05-13T13:09:31.37844353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:32.14914964Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T13:09:35.610238285Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T13:09:36.379218556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:09:37.376768898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:09:37.384354365Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:09:39.072140257Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-05-13T13:09:41.379293775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:09:41.379327802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:09:41.379337266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:09:41.379327771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:42.533840596Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
ts=2026-05-13T13:09:43.37798689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35974"
ts=2026-05-13T13:09:44.022580435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:09:45.995225422Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-13T13:09:46.37883489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:49.456632165Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:09:51.378748579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:52.918005916Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-13T13:09:53.774576519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:09:53.784149865Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:09:53.784198116Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:09:53.784239238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:09:53.790625667Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:09:55.151995928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:09:56.378989746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:09:56.380119592Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-13T13:09:59.841804623Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
ts=2026-05-13T13:10:00.327327962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36590"
level=warn ts=2026-05-13T13:10:01.379273047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:03.302721737Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-13T13:10:05.264008111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:10:06.378646072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:06.766791863Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=debug ts=2026-05-13T13:10:06.76682262Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-05-13T13:10:10.226376051Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:10:10.483491576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="261 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:10:10.491029525Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T13:10:11.359825063Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:10:11.359834353Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:10:11.378325668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:10:11.378346246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:10:11.378354516Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:10:11.378389931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:13.687495608Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T13:10:14.024257007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:10:16.139078362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41268"
level=warn ts=2026-05-13T13:10:16.379243362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:17.148712472Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-13T13:10:20.610367042Z caller=checkpoint.go:372 msg="writing series" size="580 kB"
level=warn ts=2026-05-13T13:10:21.378972199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:24.072546405Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T13:10:26.379147313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:27.533427816Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-13T13:10:30.009620333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:10:30.994838033Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:10:31.378772397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:34.457026801Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T13:10:36.378619547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:10:37.259748698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:10:37.269783192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:10:37.918278765Z caller=checkpoint.go:372 msg="writing series" size="584 kB"
level=debug ts=2026-05-13T13:10:37.918301764Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-05-13T13:10:41.378680229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:10:41.378712166Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:10:41.378723087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:10:41.378749667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:41.380164501Z caller=checkpoint.go:372 msg="writing series" size="705 kB"
ts=2026-05-13T13:10:44.026091948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:10:44.844317646Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-13T13:10:46.379392528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:48.30250933Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T13:10:49.331485174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="261 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:10:49.350004994Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:10:51.378462054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:51.765032579Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=debug ts=2026-05-13T13:10:51.765060296Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-13T13:10:55.226116479Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=warn ts=2026-05-13T13:10:56.378624041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:10:58.687550989Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=debug ts=2026-05-13T13:10:59.59286359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:11:00.330204242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59538"
level=warn ts=2026-05-13T13:11:01.379195599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:02.149609397Z caller=checkpoint.go:372 msg="writing series" size="593 kB"
level=debug ts=2026-05-13T13:11:05.611226689Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T13:11:05.611248621Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-13T13:11:06.378474182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:07.049231605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:11:09.072494016Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-13T13:11:11.359415888Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:11:11.360487969Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T13:11:11.379030606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:11:11.37905823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:11:11.379078074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:11:11.379085996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:11:12.533597938Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
ts=2026-05-13T13:11:14.027986211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:11:15.995335487Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-13T13:11:16.144332987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52560"
level=warn ts=2026-05-13T13:11:16.379328206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:19.456551664Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T13:11:21.379233892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:22.918426002Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-13T13:11:26.378653554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:26.380454614Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T13:11:26.3804699Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
level=debug ts=2026-05-13T13:11:29.841597244Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
ts=2026-05-13T13:11:30.332045538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59060"
level=warn ts=2026-05-13T13:11:31.378420342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:32.354696867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:11:32.354758243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:11:32.363857389Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:11:33.302894026Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=info ts=2026-05-13T13:11:34.669684335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:11:34.676624354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:11:36.378643165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:36.764345372Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T13:11:40.225916964Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=info ts=2026-05-13T13:11:41.378968379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:11:41.378995248Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:11:41.379005536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:11:41.379020348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:11:42.693158434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:11:42.700456513Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:11:43.365868324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:11:43.372316887Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:11:43.68724387Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
ts=2026-05-13T13:11:44.02970538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T13:11:46.378452696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:47.149621182Z caller=checkpoint.go:372 msg="writing series" size="543 kB"
level=debug ts=2026-05-13T13:11:47.149644668Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=debug ts=2026-05-13T13:11:50.65824767Z caller=checkpoint.go:372 msg="writing series" size="37 MB"
level=debug ts=2026-05-13T13:11:50.658272729Z caller=checkpoint.go:384 msg="flushing series" totalSize="37 MB" series=1
level=warn ts=2026-05-13T13:11:51.379018664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:54.07161457Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T13:11:56.37875171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:11:57.534257551Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:12:00.379340964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:12:00.99515105Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=debug ts=2026-05-13T13:12:01.136506037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:12:01.378823179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:04.457205479Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T13:12:06.37904052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:07.917898345Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T13:12:11.360296752Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:12:11.360780533Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:12:11.379302056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:12:11.379319837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:12:11.379327616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:12:11.379340463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:11.381473879Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-13T13:12:11.381490358Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
ts=2026-05-13T13:12:14.030805241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:12:14.841734516Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-13T13:12:16.163756424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:12:16.172307662Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:12:16.378996816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:12:17.383143177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:12:17.39115863Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:12:18.302591341Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:12:21.379161726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:21.76395859Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-13T13:12:25.226262585Z caller=checkpoint.go:372 msg="writing series" size="781 kB"
level=debug ts=2026-05-13T13:12:25.226292647Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T13:12:26.379109911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:12:27.181682939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:12:27.189752955Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:12:28.687425413Z caller=checkpoint.go:372 msg="writing series" size="823 kB"
level=warn ts=2026-05-13T13:12:31.379222167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:32.037119145Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:12:32.153214584Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-05-13T13:12:32.153238472Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.1 MB" series=2
level=debug ts=2026-05-13T13:12:35.610975126Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-13T13:12:36.379317207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:39.073322614Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T13:12:39.073354446Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-13T13:12:41.379157873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:12:41.379195989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:12:41.37920372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:12:41.379253288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:42.533285145Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-13T13:12:44.031785753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:12:45.994754321Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T13:12:46.378751056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:49.456414712Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-13T13:12:51.379309448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:52.918545568Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-13T13:12:55.298377387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:12:55.300428682Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T13:12:56.378903378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:12:56.380058515Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-13T13:12:58.637794489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:12:58.645096Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:12:59.840814862Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-13T13:13:01.379349197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:01.68047617Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:13:03.302428997Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T13:13:06.378568978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:06.764697817Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=info ts=2026-05-13T13:13:07.379616256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:13:07.387110474Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:13:10.225823468Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-13T13:13:11.359731685Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:13:11.359923545Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:13:11.378377695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:13:11.378396744Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:13:11.378405179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:13:11.378405395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:13:13.387054234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56782"
level=debug ts=2026-05-13T13:13:13.688705468Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
ts=2026-05-13T13:13:14.033414964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:13:15.553862548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:13:15.561161566Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:13:15.561199662Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:13:15.561228879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:13:15.567677793Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T13:13:16.149770623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:45304"
level=warn ts=2026-05-13T13:13:16.378649339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:17.148904556Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T13:13:20.610249968Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-13T13:13:21.379082308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:24.072700927Z caller=checkpoint.go:372 msg="writing series" size="700 kB"
level=debug ts=2026-05-13T13:13:24.072763255Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=13
level=warn ts=2026-05-13T13:13:26.3784309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:27.533225425Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:13:29.284016324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:13:29.29502795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T13:13:30.338723576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48912"
level=debug ts=2026-05-13T13:13:30.995318797Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:13:31.379369436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:33.870218003Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:13:34.456341294Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=warn ts=2026-05-13T13:13:36.378884692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:13:37.918152991Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-13T13:13:41.378376826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:13:41.378410382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:13:41.378445875Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:13:41.378455128Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:13:41.379524259Z caller=checkpoint.go:384 msg="flushing series" totalSize="694 kB" series=4
level=info ts=2026-05-13T13:13:43.201216573Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-13T13:13:43.204189482Z caller=checkpoint.go:670 msg="checkpoint done" time=4m31.82507634s
ts=2026-05-13T13:13:44.034529501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T13:13:46.37903576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:13:51.378541096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:13:53.572717312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:13:53.579635601Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:13:55.444567734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:13:56.379365787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:14:01.379055652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:14:01.52744939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:14:01.53344278Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:14:02.330974044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:14:02.337727622Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:14:03.477954938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:14:06.378796011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:11.359414945Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:14:11.360449474Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:14:11.378986527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:14:11.379014086Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:14:11.379024975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:14:11.379022946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:14:11.379034416Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:14:11.37916131Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-13T13:14:11.379171025Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:14:11.379209954Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.981µs success=true
level=debug ts=2026-05-13T13:14:11.379388676Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T13:14:14.035733094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:14:14.754912835Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-13T13:14:16.378902572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:18.130602875Z caller=checkpoint.go:372 msg="writing series" size="335 kB"
level=warn ts=2026-05-13T13:14:21.37838393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:21.504788117Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:14:21.504810406Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T13:14:24.879952955Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=warn ts=2026-05-13T13:14:26.37904923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:28.254609352Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T13:14:30.341951594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53284"
level=debug ts=2026-05-13T13:14:31.33172414Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:14:31.378973999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:31.630030891Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T13:14:35.00500634Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T13:14:36.378991831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:36.461375475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:14:38.380440939Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-13T13:14:41.378951836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:14:41.378956703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:14:41.37897786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:14:41.3789865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:14:41.755057351Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-13T13:14:41.839329687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:14:41.853341176Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T13:14:44.037299432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:14:44.199734839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:14:44.206955122Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:14:45.130449498Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T13:14:46.37918755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:48.504780131Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:14:51.379438053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:51.879490692Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-13T13:14:52.110477072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:14:52.119050924Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:14:55.25513822Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T13:14:56.379248032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:14:58.629901169Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-13T13:15:01.378877021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:02.005209058Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-13T13:15:05.380500408Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:15:05.380524001Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=13
level=warn ts=2026-05-13T13:15:06.379013214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:08.754790477Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T13:15:10.022661268Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:15:10.022725637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:15:10.030476905Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:15:11.359384224Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:15:11.360427924Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:15:11.36082023Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T13:15:11.360863108Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976308 n=1
level=debug ts=2026-05-13T13:15:11.360902677Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976308 n=1
level=info ts=2026-05-13T13:15:11.379130172Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:15:11.379149002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:15:11.379144032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:15:11.379155536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:15:12.130415957Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T13:15:14.03849667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:15:15.50456184Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-05-13T13:15:16.378970226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:17.184867124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:15:18.87996279Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=warn ts=2026-05-13T13:15:21.379245508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:22.255325804Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T13:15:25.629893695Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=debug ts=2026-05-13T13:15:25.629914275Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-13T13:15:26.37840745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:15:28.419880891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:15:28.427342602Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:15:29.004961362Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T13:15:30.216236563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:15:30.223120891Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T13:15:30.344745834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:47394"
level=warn ts=2026-05-13T13:15:31.379089524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:32.380229133Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T13:15:35.755515341Z caller=checkpoint.go:372 msg="writing series" size="744 kB"
level=debug ts=2026-05-13T13:15:35.86552876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:15:36.378594533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:15:37.691860664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:15:37.698615546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:15:39.130810756Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T13:15:39.130831017Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-05-13T13:15:40.30545784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:15:40.312488028Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:15:41.379272832Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:15:41.379282116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:41.379297615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:15:41.379307283Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:15:42.505130867Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
ts=2026-05-13T13:15:44.040374011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:15:45.880196293Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:15:46.379026714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:49.255586209Z caller=checkpoint.go:372 msg="writing series" size="504 kB"
level=warn ts=2026-05-13T13:15:51.379276753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:52.629789897Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:15:56.005346719Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-13T13:15:56.37900785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:15:57.104677295Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:15:59.380595223Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-13T13:15:59.380614087Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-13T13:16:00.345823009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54118"
level=warn ts=2026-05-13T13:16:01.379231858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:16:01.495593649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:16:01.502863925Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:16:02.755155863Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-13T13:16:04.063540831Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:16:06.131926212Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T13:16:06.13194972Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=2
level=warn ts=2026-05-13T13:16:06.378651712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:09.505162439Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=debug ts=2026-05-13T13:16:11.359211999Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:16:11.360279762Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:16:11.378781466Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:16:11.378785179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:11.378796183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:16:11.378803185Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:16:12.880189694Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
ts=2026-05-13T13:16:14.041836132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:16:16.157911672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:46406"
level=debug ts=2026-05-13T13:16:16.255457915Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T13:16:16.378640109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:19.629975473Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T13:16:21.378394706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:23.004877957Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=warn ts=2026-05-13T13:16:26.379384322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:26.380277881Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T13:16:26.380296493Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=6
level=info ts=2026-05-13T13:16:26.538521685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:16:26.546905647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:16:26.546932198Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:16:26.546970498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:16:26.554077452Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:16:29.755406115Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
ts=2026-05-13T13:16:30.346789616Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41068"
level=info ts=2026-05-13T13:16:30.794451946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:16:30.801052222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:16:31.378594216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:33.006415924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:16:33.12983991Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=warn ts=2026-05-13T13:16:36.37906569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:36.505420234Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T13:16:38.35190588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:16:38.353698674Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T13:16:38.802061388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:16:38.80894431Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:16:39.879503249Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-13T13:16:41.378521737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:16:41.378547171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:16:41.378555189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:16:41.378586777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:43.255209796Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
ts=2026-05-13T13:16:43.396091631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53738"
level=info ts=2026-05-13T13:16:43.781593282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:16:43.78831561Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T13:16:44.04326442Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:16:46.378994658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:46.63044604Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-13T13:16:46.630465203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-13T13:16:50.052220591Z caller=checkpoint.go:372 msg="writing series" size="42 MB"
level=debug ts=2026-05-13T13:16:50.05224724Z caller=checkpoint.go:384 msg="flushing series" totalSize="42 MB" series=1
level=warn ts=2026-05-13T13:16:51.37918708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:53.379917147Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-13T13:16:56.181229588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:16:56.191622055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:16:56.378890152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:16:56.754745004Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:17:00.055053809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:17:00.130304784Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
ts=2026-05-13T13:17:00.348170343Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:46694"
level=warn ts=2026-05-13T13:17:01.378991794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:03.505322775Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T13:17:06.378767112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:06.880153236Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:17:10.257938256Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-13T13:17:10.257960137Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=6
level=info ts=2026-05-13T13:17:11.359655913Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:17:11.360033647Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:17:11.378504319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:17:11.378534452Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:17:11.378543227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:17:11.37856301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:13.62962361Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T13:17:13.928959777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:17:13.93703281Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T13:17:14.044630468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:17:16.160401389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41922"
level=warn ts=2026-05-13T13:17:16.379105905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:17.005477837Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:17:20.379936342Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-13T13:17:21.378981079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:17:22.189746477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:17:22.197287533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:17:23.75563011Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=warn ts=2026-05-13T13:17:26.378699108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:27.130561089Z caller=checkpoint.go:372 msg="writing series" size="822 kB"
level=debug ts=2026-05-13T13:17:27.130583194Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=debug ts=2026-05-13T13:17:29.122737849Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:17:30.349270268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:44102"
level=debug ts=2026-05-13T13:17:30.511741845Z caller=checkpoint.go:372 msg="writing series" size="7.9 MB"
level=debug ts=2026-05-13T13:17:30.511762838Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.9 MB" series=1
level=warn ts=2026-05-13T13:17:31.379047936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:33.880081984Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T13:17:35.916166714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:17:35.924840986Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:17:36.378852222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:36.992021955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:17:37.255018523Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-13T13:17:40.631534375Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T13:17:40.631559796Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=info ts=2026-05-13T13:17:41.378648515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:17:41.37865756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:41.378673707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:17:41.37868199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:17:44.005080567Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
ts=2026-05-13T13:17:44.046405921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:17:46.162148979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:32806"
level=warn ts=2026-05-13T13:17:46.379441126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:47.380030751Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T13:17:50.754542739Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-13T13:17:51.113417387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:17:51.124110379Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:17:51.378870048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:54.129989023Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T13:17:56.378987225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:17:57.504907154Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-13T13:18:00.879506688Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-13T13:18:01.378479398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:04.254436118Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=debug ts=2026-05-13T13:18:04.761130537Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:18:06.379271877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:07.630115114Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=info ts=2026-05-13T13:18:08.418311913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:18:08.426223211Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:18:09.584579675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:18:09.591332773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:18:11.004670719Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-13T13:18:11.359629357Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:18:11.360338576Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:18:11.378838424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:18:11.378856546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:18:11.378864244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:18:11.378870575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:18:14.047531337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:18:14.380326187Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-13T13:18:15.879771736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:18:16.379401352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:17.755193631Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T13:18:21.129510443Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-13T13:18:21.37886235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:18:22.786023514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:18:22.795735568Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:18:22.795754248Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:18:22.795776914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:18:22.802704892Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:18:24.505654868Z caller=checkpoint.go:372 msg="writing series" size="739 kB"
level=debug ts=2026-05-13T13:18:24.505676118Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=13
level=warn ts=2026-05-13T13:18:26.37900655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:27.880136194Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-13T13:18:30.352735679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:46370"
level=debug ts=2026-05-13T13:18:31.255097778Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:18:31.378686152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:18:31.650702783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:18:31.657592765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:18:33.006560134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:18:34.630319994Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-13T13:18:36.378919769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:38.004717801Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:18:41.062186792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:18:41.072905793Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:18:41.378616171Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:18:41.378644459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:18:41.378656093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:18:41.378661246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:41.379747834Z caller=checkpoint.go:384 msg="flushing series" totalSize="566 kB" series=4
level=info ts=2026-05-13T13:18:41.383725812Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-13T13:18:41.385773233Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006424834s
ts=2026-05-13T13:18:44.049427003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:18:46.165159427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38078"
level=warn ts=2026-05-13T13:18:46.378992438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:18:51.378793253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:18:51.951798547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:18:51.975331931Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:18:56.379055062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:18:57.612146201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:19:00.353542133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:57082"
level=warn ts=2026-05-13T13:19:01.379044178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:19:03.600564416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:19:03.611974517Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:19:06.378794323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:19:07.139255359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:19:07.148393496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:19:11.359731413Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:19:11.359756647Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:19:11.37830852Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:19:11.378379455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:19:11.378376555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:11.378396769Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:19:11.378404143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:19:11.37841217Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-13T13:19:11.378419763Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:19:11.37843486Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.46µs success=true
level=debug ts=2026-05-13T13:19:11.378584656Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T13:19:13.402053236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:49758"
ts=2026-05-13T13:19:14.050471426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:19:14.754636149Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-13T13:19:16.379149693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:18.105358774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:19:18.129794875Z caller=checkpoint.go:372 msg="writing series" size="448 kB"
level=warn ts=2026-05-13T13:19:21.379284035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:21.504708443Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:19:21.504732555Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-13T13:19:24.87910796Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=warn ts=2026-05-13T13:19:26.378421566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:28.254123461Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T13:19:29.74616063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:19:29.75399331Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:19:30.931294797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:19:30.931345696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="499 kB" avg_comp="499 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:19:30.935296424Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T13:19:31.378544266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:31.629493697Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T13:19:35.004234132Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T13:19:36.37892026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:38.379368556Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=info ts=2026-05-13T13:19:40.257739941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="331 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:19:40.265282444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T13:19:41.378615944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:19:41.37862325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:41.378638356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:19:41.378645676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:19:41.754339688Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-13T13:19:42.263029665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:19:42.267016106Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:19:44.051788012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:19:45.12969766Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-13T13:19:46.378986707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:19:47.971742558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:19:47.979662882Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:19:48.50410929Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:19:51.378377481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:19:51.878719078Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T13:19:55.253964894Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T13:19:56.379423928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:19:57.232960609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:19:57.24533752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:19:57.245368993Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:19:57.2453948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:19:57.252315764Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:19:58.629716638Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-13T13:19:58.62973672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
ts=2026-05-13T13:20:00.356730843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56344"
level=warn ts=2026-05-13T13:20:01.37879372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:02.004128477Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-13T13:20:04.015749143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:20:05.37936646Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:20:05.781815022Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:20:06.379230258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:08.75394335Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T13:20:08.753965812Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-13T13:20:11.359676528Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:20:11.359859872Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:20:11.378328765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:20:11.378345082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:20:11.378352248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:20:11.378384012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:12.129442481Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T13:20:13.405123798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38468"
ts=2026-05-13T13:20:14.053279105Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:20:15.503945183Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=warn ts=2026-05-13T13:20:16.378972904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:20:16.723659114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:20:16.730300397Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:20:18.156151883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:20:18.161529143Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:20:18.879808648Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=warn ts=2026-05-13T13:20:21.379208672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:22.254649894Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T13:20:25.629530821Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=warn ts=2026-05-13T13:20:26.37870461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:20:27.377158881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:20:27.383749902Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:20:29.003996769Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T13:20:30.357688149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:35874"
level=warn ts=2026-05-13T13:20:31.378988565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:32.379368174Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-13T13:20:33.020813722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:20:33.027885187Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T13:20:34.568429808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:20:34.577150485Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:20:35.754714529Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=debug ts=2026-05-13T13:20:35.754737964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=8
level=warn ts=2026-05-13T13:20:36.379048462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:38.601734119Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:20:39.129876688Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T13:20:39.129899899Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=info ts=2026-05-13T13:20:41.378497704Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:20:41.37852315Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:20:41.378540355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:20:41.378543696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:42.504553728Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-13T13:20:43.406816394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:55404"
ts=2026-05-13T13:20:44.054825385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:20:45.879649502Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:20:46.378915635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:20:48.465059885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:20:48.473805488Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:20:49.254383037Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=warn ts=2026-05-13T13:20:51.378880225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:52.628976738Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:20:56.005369702Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T13:20:56.005392721Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=debug ts=2026-05-13T13:20:56.067212283Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:20:56.068862861Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T13:20:56.378847815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:20:59.379197673Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=warn ts=2026-05-13T13:21:01.378767972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:02.754198625Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-13T13:21:04.139724933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:21:04.145040772Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:21:06.132149543Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:21:06.132169734Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=warn ts=2026-05-13T13:21:06.379033667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:09.504360481Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-13T13:21:10.748748525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:21:10.75747351Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:21:11.359637464Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:21:11.359860955Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:21:11.378328975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:21:11.378359516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:21:11.37837183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:21:11.378386376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:12.879575821Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-13T13:21:14.008803673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:21:14.05622964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:21:16.254237955Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T13:21:16.378806905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:21:17.579732967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:21:17.586885986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:21:19.62955361Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T13:21:21.379230488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:23.003803519Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-13T13:21:26.379280516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:26.380639869Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T13:21:26.380659938Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=6
level=debug ts=2026-05-13T13:21:29.753980174Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-13T13:21:30.823762647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:21:30.833743307Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:21:31.379124835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:33.129841354Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-13T13:21:34.19760882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:21:34.204358741Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:21:36.379007984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:36.504411462Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T13:21:39.879226128Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-13T13:21:41.379284539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:21:41.379311618Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:21:41.379311429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:21:41.379324334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:21:43.254689374Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-13T13:21:44.057199997Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T13:21:46.379166993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:46.629464934Z caller=checkpoint.go:372 msg="writing series" size="407 kB"
level=debug ts=2026-05-13T13:21:46.629485524Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-13T13:21:48.619786265Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:21:49.294319376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:21:50.050801446Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-13T13:21:50.050826558Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=1
level=warn ts=2026-05-13T13:21:51.378779098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:53.378763334Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T13:21:56.37868684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:21:56.754361533Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:22:00.12903375Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-13T13:22:00.307648049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:22:00.319222783Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:22:00.319248135Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:22:00.319272025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:22:00.326389658Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:22:00.360654195Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:39788"
level=warn ts=2026-05-13T13:22:01.378790483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:22:01.855233488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:22:01.863044445Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T13:22:03.033968956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:22:03.039952666Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:22:03.504439307Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T13:22:06.379475525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:06.879626852Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:22:10.257427231Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-13T13:22:10.257449412Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=6
level=info ts=2026-05-13T13:22:11.3596837Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:22:11.360421355Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:22:11.378854534Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:22:11.37887073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:22:11.378878269Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:22:11.378882004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:13.628829057Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-13T13:22:13.895446831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:22:13.905152587Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:22:14.057981446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:22:14.320266707Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:22:16.174436424Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42406"
level=warn ts=2026-05-13T13:22:16.378520563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:17.003803633Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:22:20.379276593Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-13T13:22:21.379241309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:23.754672548Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:22:23.754694465Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T13:22:26.37889096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:27.129291817Z caller=checkpoint.go:372 msg="writing series" size="809 kB"
level=info ts=2026-05-13T13:22:28.721584223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:22:28.731637386Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:22:30.508548601Z caller=checkpoint.go:372 msg="writing series" size="8.4 MB"
level=debug ts=2026-05-13T13:22:30.508570548Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.2 MB" series=2
level=warn ts=2026-05-13T13:22:31.379263176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:22:33.711466257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:22:33.717871094Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:22:33.879489547Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-05-13T13:22:36.379050071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:37.253786393Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-13T13:22:38.81508356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:22:40.630576951Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T13:22:40.630599083Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-13T13:22:41.379020635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:22:41.379037682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:41.379046858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:22:41.379061876Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:22:43.51534229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:22:43.52281125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:22:44.004341097Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
ts=2026-05-13T13:22:44.059758316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:22:44.358350033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:22:44.364584247Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:22:44.364600803Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:22:44.364630202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:22:44.371401031Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:22:46.378766459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:47.379152308Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T13:22:50.754115877Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-13T13:22:51.140726988Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:22:51.378539348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:54.128965459Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-13T13:22:55.751579461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:22:55.757631027Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T13:22:56.379348691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:22:57.504500196Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
ts=2026-05-13T13:23:00.363580282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:34174"
level=debug ts=2026-05-13T13:23:00.879708043Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-13T13:23:01.379228591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:23:03.647055855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:23:03.655587478Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:23:04.255766617Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T13:23:06.37945615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:23:07.628798077Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=debug ts=2026-05-13T13:23:08.434074559Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:23:11.003795998Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-13T13:23:11.359462601Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:23:11.360503086Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:23:11.378866985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:23:11.378881405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:23:11.37888912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:23:11.3789062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:23:14.06093247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:23:14.379724008Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-13T13:23:16.379027922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:23:17.754540797Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T13:23:21.129555376Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-13T13:23:21.379309633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:23:24.504473172Z caller=checkpoint.go:372 msg="writing series" size="623 kB"
level=debug ts=2026-05-13T13:23:24.504494812Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=13
level=warn ts=2026-05-13T13:23:26.378704281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:23:27.878940218Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:23:28.233950762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:23:28.244879092Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
ts=2026-05-13T13:23:30.364953718Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42380"
level=info ts=2026-05-13T13:23:30.526488164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:23:30.533321033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:23:31.254576681Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:23:31.378749677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:23:31.718692039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:23:31.72497213Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:23:34.629617581Z caller=checkpoint.go:372 msg="writing series" size="472 kB"
level=warn ts=2026-05-13T13:23:36.379375321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:23:37.505921694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:23:38.004340884Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:23:38.72515195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:23:38.731010372Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:23:40.286442374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:23:40.289985703Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:23:40.848753295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:23:41.378355515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:23:41.378384688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:23:41.378394478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:23:41.378389963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:23:41.379360733Z caller=checkpoint.go:384 msg="flushing series" totalSize="724 kB" series=4
level=info ts=2026-05-13T13:23:41.385409741Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-13T13:23:41.3875003Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008950112s
ts=2026-05-13T13:23:44.061918853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:23:46.178500299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54748"
level=warn ts=2026-05-13T13:23:46.37892723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:23:51.378727582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:23:56.3791203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:24:00.365922276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53268"
level=warn ts=2026-05-13T13:24:01.378656403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:04.979662493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:24:06.379400964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:24:11.205534493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:24:11.213800373Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:24:11.359653962Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:24:11.35991179Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:24:11.378276904Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:24:11.378539101Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-13T13:24:11.378548424Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:24:11.378565498Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.384µs success=true
level=warn ts=2026-05-13T13:24:11.378561019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:24:11.378583717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:24:11.378602382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:24:11.378609708Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:24:11.378726385Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
ts=2026-05-13T13:24:14.06365757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:24:14.754595807Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
ts=2026-05-13T13:24:16.180093831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48574"
level=warn ts=2026-05-13T13:24:16.37915024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:18.129268349Z caller=checkpoint.go:372 msg="writing series" size="380 kB"
level=warn ts=2026-05-13T13:24:21.378675451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:21.50476483Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:24:21.50478473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T13:24:23.952627974Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:24:24.879070857Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=info ts=2026-05-13T13:24:24.953220018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:24:24.960514554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:24:25.099779949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:24:25.105879965Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:24:26.378708279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:28.254103288Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T13:24:28.762858664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:24:28.770339448Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:24:31.379278269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:31.629589333Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T13:24:35.004737702Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T13:24:36.379333924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:38.379389807Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-13T13:24:38.808590498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:24:40.242703077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:24:40.252271514Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:24:41.378983536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:24:41.379001921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:41.37901696Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:24:41.37902633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:24:41.754880673Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
ts=2026-05-13T13:24:43.41698483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38284"
ts=2026-05-13T13:24:44.065462237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:24:45.12919654Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:24:46.379344382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:48.504113892Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:24:51.379115258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:51.879614157Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:24:55.254436907Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=info ts=2026-05-13T13:24:55.961579795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:24:55.970433326Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:24:55.970454462Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:24:55.970494533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:24:55.977896425Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:24:56.378396863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:24:58.629672523Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
ts=2026-05-13T13:25:00.369082502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:43138"
level=warn ts=2026-05-13T13:25:01.379169089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:02.004110902Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T13:25:05.379975148Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:25:05.379998993Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=13
level=warn ts=2026-05-13T13:25:06.379392155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:06.83780131Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:25:06.839754463Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:25:08.257896233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:25:08.754367453Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:25:09.706944127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:25:09.715871526Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:25:09.848375655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:25:09.854814628Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:25:11.359497855Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:25:11.360560442Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:25:11.378932006Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:25:11.378947744Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:25:11.378955253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:25:11.37895162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:12.129118087Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T13:25:14.066923789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:25:14.361408397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:25:14.368493198Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:25:15.504617646Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
ts=2026-05-13T13:25:16.182802565Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59852"
level=warn ts=2026-05-13T13:25:16.37863993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:18.880002168Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=warn ts=2026-05-13T13:25:21.378383329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:22.254493967Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T13:25:25.629901998Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=info ts=2026-05-13T13:25:25.780289948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:25:25.786170582Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:25:26.010757018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:25:26.017737592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:25:26.378826396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:29.004021161Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T13:25:30.370195802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49716"
level=warn ts=2026-05-13T13:25:31.378508544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:32.379462745Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T13:25:35.754984131Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T13:25:35.755005109Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=warn ts=2026-05-13T13:25:36.379035392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:25:36.784389914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:25:36.793865341Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:25:38.615971487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:25:39.130191752Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T13:25:39.130213842Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-05-13T13:25:41.379279907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:25:41.379286754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:41.379301514Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:25:41.379308958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:25:42.504349316Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-13T13:25:44.067941847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:25:45.879744919Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:25:46.378922259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:49.254266934Z caller=checkpoint.go:372 msg="writing series" size="634 kB"
level=warn ts=2026-05-13T13:25:51.378421756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:25:52.629546492Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:25:56.005513104Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T13:25:56.005535306Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=5
level=warn ts=2026-05-13T13:25:56.378982689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:25:56.608123296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:25:56.617636672Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:25:59.379964751Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=warn ts=2026-05-13T13:26:01.378643067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:02.754101283Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-13T13:26:03.059643211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:26:06.131765486Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:26:06.131788963Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=warn ts=2026-05-13T13:26:06.378530713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:09.504027533Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-13T13:26:09.696565366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:26:09.704073341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:26:11.224301692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:26:11.359049813Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:26:11.36009569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:26:11.378404679Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:26:11.378421413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:26:11.378429799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:26:11.378473177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:12.879065961Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
ts=2026-05-13T13:26:14.069584694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:26:16.185935106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59132"
level=debug ts=2026-05-13T13:26:16.254447981Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T13:26:16.378791523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:19.628940128Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T13:26:21.378693812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:26:21.987216429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:26:21.994490481Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:26:21.994508336Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:26:21.994531259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:26:21.997739167Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:26:23.004288178Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-13T13:26:24.484760224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:26:24.491274508Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T13:26:26.378756031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:26.379869159Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T13:26:26.379882954Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
level=debug ts=2026-05-13T13:26:29.754768916Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-13T13:26:30.14744474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:26:30.155848462Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:26:31.379092792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:26:32.605626015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:26:32.609413035Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:26:33.129919484Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=warn ts=2026-05-13T13:26:36.378587437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:36.503955491Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T13:26:36.87390379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:26:36.880485353Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:26:39.878918473Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-13T13:26:41.378971092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:26:41.378996969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:26:41.378987518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:26:41.379005472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:26:42.037330014Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:26:43.253992475Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-05-13T13:26:43.422513299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51274"
ts=2026-05-13T13:26:44.070930204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:26:46.187034766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41650"
level=warn ts=2026-05-13T13:26:46.37919882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:46.629756021Z caller=checkpoint.go:372 msg="writing series" size="473 kB"
level=debug ts=2026-05-13T13:26:46.629775322Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-13T13:26:47.852895157Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:26:50.049650698Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-13T13:26:50.049680141Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=1
level=warn ts=2026-05-13T13:26:51.378825268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:26:53.379389029Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T13:26:56.379184564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:26:56.392322531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:26:56.398272438Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:26:56.398290698Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:26:56.398314913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:26:56.40349212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:26:56.754498851Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:27:00.129750162Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-13T13:27:01.378704798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:27:02.589277255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:27:02.594887822Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:27:03.504416234Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T13:27:06.379233632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:06.87963554Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:27:10.255802342Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-13T13:27:10.255826052Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=6
level=info ts=2026-05-13T13:27:11.359750871Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:27:11.360149942Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:27:11.378459394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:27:11.378463344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:11.378476582Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:27:11.378486033Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:27:13.629223048Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=info ts=2026-05-13T13:27:13.756655842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:27:13.765170456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:27:13.976762206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:27:13.984748557Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T13:27:14.072208276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:27:14.497030381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:27:14.504550236Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:27:15.051225258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:27:16.379056636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:17.004587437Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:27:20.379657945Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-13T13:27:21.378903832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:23.75535443Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:27:23.755376405Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-13T13:27:26.378791608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:27.130139507Z caller=checkpoint.go:372 msg="writing series" size="868 kB"
level=info ts=2026-05-13T13:27:28.836468693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:27:28.847947226Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:27:30.376195546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:35654"
level=debug ts=2026-05-13T13:27:30.512913326Z caller=checkpoint.go:372 msg="writing series" size="9.3 MB"
level=debug ts=2026-05-13T13:27:30.512936177Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=2
level=warn ts=2026-05-13T13:27:31.379112322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:33.879275433Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T13:27:36.378955022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:37.254448777Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-13T13:27:37.92933082Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:27:39.139888667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:27:40.63100757Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T13:27:40.631029202Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=info ts=2026-05-13T13:27:41.155982962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:27:41.165245229Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:27:41.378399573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:27:41.378424153Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:27:41.378437937Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:27:41.378476393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:27:43.425652151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42994"
level=debug ts=2026-05-13T13:27:44.004768202Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
ts=2026-05-13T13:27:44.073255695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:27:46.379229725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:47.3798947Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T13:27:50.753920306Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-13T13:27:51.379022478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:54.129212684Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T13:27:56.379204824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:27:57.504274378Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-13T13:27:59.128616226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:27:59.133611439Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:28:00.3774442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54378"
level=debug ts=2026-05-13T13:28:00.879769941Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-13T13:28:01.379123406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:28:02.147709435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:28:02.155301926Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:28:04.254519236Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T13:28:06.378793596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:28:07.629510417Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=info ts=2026-05-13T13:28:10.314962099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:28:10.322213562Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:28:10.322232906Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:28:10.322255848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:28:10.328336046Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:28:11.004477847Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-13T13:28:11.359102433Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:28:11.360153971Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T13:28:11.378420612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:28:11.378423086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:28:11.378441672Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:28:11.378451296Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:28:12.768242018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:28:12.774951696Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T13:28:13.766924637Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:28:13.76699957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:28:13.772440025Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:28:14.074415107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:28:14.379311997Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-13T13:28:16.379137088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:28:17.753945538Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T13:28:21.128888051Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-13T13:28:21.3191942Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:28:21.379322666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:28:24.285417552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:28:24.292667595Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:28:24.504936055Z caller=checkpoint.go:372 msg="writing series" size="624 kB"
level=debug ts=2026-05-13T13:28:24.504957595Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=13
level=info ts=2026-05-13T13:28:26.19822429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:28:26.204154564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:28:26.379342278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:28:27.879718814Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:28:28.735169435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:28:28.742768778Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:28:30.37872674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:52312"
level=debug ts=2026-05-13T13:28:31.254712889Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:28:31.379023496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:28:34.629556099Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=warn ts=2026-05-13T13:28:36.378849448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:28:38.004445686Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-13T13:28:41.379062957Z caller=checkpoint.go:384 msg="flushing series" totalSize="804 kB" series=4
level=info ts=2026-05-13T13:28:41.379102295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:28:41.379124914Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:28:41.379121241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:28:41.379137125Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:28:41.386227841Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-13T13:28:41.388812326Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.0101205s
level=info ts=2026-05-13T13:28:43.916701426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:28:43.922579403Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:28:44.076314879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:28:45.386036302Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:28:46.378900233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:28:49.11590531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:28:49.123071859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:28:49.569619705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:28:49.576279038Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:28:51.379126488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:28:56.378796185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:28:56.594084781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:28:56.599709954Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:29:00.380536062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:40486"
level=warn ts=2026-05-13T13:29:01.378694471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:29:06.378678236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:29:06.866306795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:29:06.876487539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T13:29:07.033801977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:29:07.041218358Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:29:09.90142787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:29:11.359388269Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:29:11.360434388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:29:11.378791669Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:29:11.378819328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:29:11.378821522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:11.378832177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:29:11.378843568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:29:11.378912132Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-13T13:29:11.378923001Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:29:11.378940942Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.679µs success=true
level=debug ts=2026-05-13T13:29:11.379127851Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-13T13:29:13.429267092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:49974"
ts=2026-05-13T13:29:14.077631506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:29:14.755567787Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-13T13:29:16.378978396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:18.130016442Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-05-13T13:29:21.09527707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:29:21.096901802Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T13:29:21.378696632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:21.505042023Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:29:24.880245652Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-13T13:29:24.880267623Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-13T13:29:26.379191382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:28.255069552Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:29:31.378878946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:31.629331014Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T13:29:35.00498712Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T13:29:36.139120411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:29:36.145780139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:29:36.269426478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:29:36.378500755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:38.379605834Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-13T13:29:41.378766558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:29:41.37879625Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:29:41.378815353Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:29:41.378824743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:29:41.603022303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:29:41.608546004Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:29:41.754699749Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-13T13:29:41.943156952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:29:41.949466287Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
ts=2026-05-13T13:29:43.430964128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47976"
ts=2026-05-13T13:29:44.079333278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:29:45.130091682Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-13T13:29:46.194633289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60004"
level=warn ts=2026-05-13T13:29:46.379221361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:48.504409168Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T13:29:49.431371512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:29:49.439026237Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:29:51.379203101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:29:51.879429075Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:29:55.254295174Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=warn ts=2026-05-13T13:29:56.379200211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:29:58.056924361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="355 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:29:58.063556671Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e215e5a27%3A19e2168a54d%3Aa4c065fb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T13:29:58.063580745Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:29:58.063603695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:29:58.068201541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:29:58.62987484Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-13T13:30:01.378414137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:01.713469962Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:30:02.005072488Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-13T13:30:02.858941762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:30:02.864990841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:30:05.380616239Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:30:05.380638487Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=12
level=warn ts=2026-05-13T13:30:06.37866662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:08.755275904Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T13:30:11.359395983Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:30:11.360451506Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:30:11.361718029Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T13:30:11.361754005Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976309 n=1
level=debug ts=2026-05-13T13:30:11.361793896Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976309 n=1
level=info ts=2026-05-13T13:30:11.378817704Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:30:11.378841079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:30:11.378852852Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:30:11.378867979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:12.129755232Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-13T13:30:14.0804741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:30:15.307385177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:30:15.505207195Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-13T13:30:16.19549833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:51324"
level=warn ts=2026-05-13T13:30:16.378916573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:30:17.763493224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:30:17.77091185Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:30:18.880561209Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=warn ts=2026-05-13T13:30:21.378828219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:22.255165961Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T13:30:25.629426165Z caller=checkpoint.go:372 msg="writing series" size="460 kB"
level=debug ts=2026-05-13T13:30:25.629449694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T13:30:26.379251653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:29.004318287Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T13:30:29.15386076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:30:29.913965773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:30:29.921872862Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T13:30:31.378674115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:32.379937967Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T13:30:35.75484647Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=warn ts=2026-05-13T13:30:36.378659036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:30:38.005750248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:30:38.011389318Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:30:38.43769152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:30:38.44287901Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:30:38.672644105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:30:38.679399399Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:30:39.130937801Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T13:30:39.130959361Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-13T13:30:41.378419676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:30:41.378442371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:30:41.378452462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:30:41.378476646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:42.504427433Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-13T13:30:43.434224512Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41082"
ts=2026-05-13T13:30:44.08173227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:30:44.896714042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:30:44.903978622Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:30:45.879474116Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:30:46.378567558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:49.254669552Z caller=checkpoint.go:372 msg="writing series" size="644 kB"
level=warn ts=2026-05-13T13:30:51.37840255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:52.629471931Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:30:56.006186535Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T13:30:56.006209925Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=warn ts=2026-05-13T13:30:56.378832431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:30:59.379758337Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T13:30:59.995039058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:31:00.006537006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:31:01.378828059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:02.75491638Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-13T13:31:03.461018394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:31:06.132945957Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:31:06.13296713Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=warn ts=2026-05-13T13:31:06.3792056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:09.504659328Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-05-13T13:31:11.36021534Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:31:11.360632796Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=warn ts=2026-05-13T13:31:11.378960585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:31:11.378971697Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:31:11.378989924Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:31:11.378998419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:31:12.879349964Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
ts=2026-05-13T13:31:14.083281779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:31:14.765315694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:31:16.078153192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:31:16.081722209Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:31:16.255054016Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T13:31:16.379156905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:19.629393609Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T13:31:21.378651041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:31:21.686521879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:31:21.692039006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:31:23.004692101Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-13T13:31:26.378744604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:26.380689106Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T13:31:26.380702186Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=6
level=info ts=2026-05-13T13:31:28.230677969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:31:28.237741478Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:31:28.237759882Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:31:28.237782327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:31:28.243882705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:31:29.754460191Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-13T13:31:30.562728893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:31:30.570949828Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:31:31.378706555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:31:31.425877389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:31:31.432115127Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:31:32.733738166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:31:33.130002813Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
level=warn ts=2026-05-13T13:31:36.378969878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:36.504376783Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T13:31:39.879430974Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-13T13:31:41.378336593Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:31:41.378367233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:31:41.378375123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:31:41.378435327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:43.254874592Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-13T13:31:43.925701433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:31:43.93689499Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
ts=2026-05-13T13:31:44.084642127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:31:46.378760543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:46.630251197Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-13T13:31:46.630270614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-13T13:31:47.690453383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.7 MB" avg_comp="2.2 MB" total_uncomp="58 MB" avg_uncomp="20 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:31:47.696650633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:31:50.056690683Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-13T13:31:50.056714111Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=1
level=warn ts=2026-05-13T13:31:51.378388936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:53.38008558Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=debug ts=2026-05-13T13:31:55.220689543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:31:56.37941274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:31:56.755508745Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:31:59.324737361Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:32:00.129992114Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-13T13:32:01.378484451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:32:03.487560695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:32:03.492440762Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:32:03.504521428Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T13:32:06.37939803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:06.880133843Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:32:10.258263369Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-13T13:32:10.258288283Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=6
level=info ts=2026-05-13T13:32:11.359714398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:32:11.359982532Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:32:11.379341604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:32:11.379358598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:32:11.379366465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:32:11.379387983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:32:12.669368978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:32:12.675414514Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:32:13.437441514Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38632"
level=debug ts=2026-05-13T13:32:13.629482732Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-05-13T13:32:13.756893091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:32:13.764872701Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
ts=2026-05-13T13:32:14.085937165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:32:14.175961061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:32:14.18177465Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-13T13:32:16.20094001Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:56678"
level=warn ts=2026-05-13T13:32:16.378914911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:17.004558762Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:32:20.379444894Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-13T13:32:21.379141669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:23.755062391Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:32:23.755084236Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-13T13:32:26.379170093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:27.129891891Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=debug ts=2026-05-13T13:32:29.879021976Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:32:30.49630354Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:32:30.512204318Z caller=checkpoint.go:372 msg="writing series" size="9.8 MB"
level=debug ts=2026-05-13T13:32:30.512218633Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-13T13:32:30.547521058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:32:30.554098442Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:32:31.378688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:33.879978244Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T13:32:36.378725674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:37.25437107Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-13T13:32:40.631586987Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T13:32:40.631607243Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=info ts=2026-05-13T13:32:41.378905286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:32:41.37892817Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:32:41.378935966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:32:41.378927178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:32:43.4382215Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39222"
level=debug ts=2026-05-13T13:32:44.004307474Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
ts=2026-05-13T13:32:44.087673146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
ts=2026-05-13T13:32:46.202550721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49238"
level=info ts=2026-05-13T13:32:46.280098614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:32:46.287610752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:32:46.378820663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:47.379600214Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-13T13:32:48.571848513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:32:48.576065479Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:32:48.576083814Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:32:48.576110834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:32:48.579119205Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:32:50.755122989Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-13T13:32:51.378608509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:54.130034392Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-13T13:32:55.51594471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:32:55.523608794Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:32:56.378934013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:32:57.50492782Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
ts=2026-05-13T13:33:00.390207996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:47500"
level=debug ts=2026-05-13T13:33:00.880262487Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-13T13:33:01.379295763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:01.689348847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:33:04.255120491Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-13T13:33:05.608391189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:33:05.615420332Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:33:06.379352431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:07.629794Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=info ts=2026-05-13T13:33:07.793299983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:33:07.79760018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:33:11.005241422Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=debug ts=2026-05-13T13:33:11.359608386Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:33:11.359634012Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:33:11.378996815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:33:11.379017688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:33:11.379025549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:33:11.379080862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:33:11.530524245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:33:11.536659672Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-13T13:33:14.089405937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:33:14.38014003Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=warn ts=2026-05-13T13:33:16.378395283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:17.755030749Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-13T13:33:19.926793107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:33:19.932048349Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:33:21.130073626Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-13T13:33:21.378814444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:21.976478826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:33:21.978241467Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:33:24.504789235Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-13T13:33:24.504814154Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=13
level=warn ts=2026-05-13T13:33:26.379256301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:27.879260536Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-13T13:33:30.391819024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:58914"
level=debug ts=2026-05-13T13:33:31.254853361Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:33:31.378798906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:34.630309174Z caller=checkpoint.go:372 msg="writing series" size="436 kB"
level=warn ts=2026-05-13T13:33:36.37864855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:37.668729335Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:33:38.004643452Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:33:38.802053948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:33:38.806691512Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:33:40.444216138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:33:40.450967291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:33:40.450986135Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:33:40.451010107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:33:40.45675552Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:33:41.378497405Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:33:41.378521319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:33:41.378535089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:33:41.378550385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:33:41.379600697Z caller=checkpoint.go:384 msg="flushing series" totalSize="689 kB" series=4
level=info ts=2026-05-13T13:33:41.385022547Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-13T13:33:41.385998166Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006903691s
ts=2026-05-13T13:33:44.090911444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:33:46.378514929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:33:51.37856433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:33:54.812321845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:33:54.82145725Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:33:55.274470509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:33:55.281064586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:33:56.379186538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:33:59.010760403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:33:59.016335499Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-13T13:34:00.393408345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59522"
level=warn ts=2026-05-13T13:34:01.378677321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:34:03.361287021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:34:03.368233998Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:34:06.378670963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:07.733578465Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:34:10.37620403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:34:11.359389109Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:34:11.36043537Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:34:11.378798473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T13:34:11.378806514Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-13T13:34:11.378816949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:34:11.378825163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:34:11.378866111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:34:11.378907302Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-13T13:34:11.37891563Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:34:11.378932542Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.267µs success=true
level=debug ts=2026-05-13T13:34:11.379088601Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-13T13:34:14.092196079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:34:14.755130454Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-13T13:34:14.929167241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:34:14.936634747Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:34:16.379243618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:34:17.549759485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:34:17.556604426Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:34:18.130145663Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=warn ts=2026-05-13T13:34:21.379276312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:21.504655913Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:34:21.504677614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T13:34:24.87985091Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=warn ts=2026-05-13T13:34:26.37921009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:34:26.932467868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:34:26.936079565Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:34:28.254967627Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:34:31.378419917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:31.629360831Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T13:34:35.004530668Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T13:34:36.379251667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:38.379134576Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-13T13:34:41.268967458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:34:41.378331015Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:34:41.37835655Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:34:41.378364371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:34:41.378389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:41.755220281Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
ts=2026-05-13T13:34:44.093611487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:34:45.130285358Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:34:46.379406601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:34:47.439457662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:34:47.446196335Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:34:48.505090644Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:34:51.378915539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:51.87952381Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-13T13:34:52.350854884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:34:52.363642666Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=info ts=2026-05-13T13:34:53.541603811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:34:53.547978823Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:34:53.547996823Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:34:53.548018266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:34:53.553602579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:34:55.254118585Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-13T13:34:56.3787491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:34:58.629440781Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-13T13:35:01.378414426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:02.004543194Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T13:35:02.004560447Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-05-13T13:35:05.380314389Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=warn ts=2026-05-13T13:35:06.378973498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:08.754324184Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:35:08.982021071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:35:08.988525701Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:35:08.98854499Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:35:08.988567432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:35:08.993170031Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:35:09.15070472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:35:09.157293287Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:35:10.082770032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:35:10.08873893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:35:11.359337474Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:35:11.360460776Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T13:35:11.378828638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:35:11.378822128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:35:11.378855094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:35:11.378865917Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:35:11.683014257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:35:12.130106712Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-13T13:35:13.30345606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:35:13.306796528Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:35:14.095340117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:35:14.953528081Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:35:15.504221539Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T13:35:16.209723345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53528"
level=warn ts=2026-05-13T13:35:16.379236424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:18.879596868Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=debug ts=2026-05-13T13:35:18.879618911Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-13T13:35:20.51581253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:35:20.521237032Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:35:21.378510167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:22.254760174Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-13T13:35:25.629650304Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=warn ts=2026-05-13T13:35:26.379106907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:29.00504783Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:35:31.378882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:32.379661812Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T13:35:35.755628055Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T13:35:35.755651462Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-13T13:35:36.378908633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:39.131057829Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T13:35:39.131077796Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-13T13:35:41.379203236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:35:41.379226989Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:35:41.379236352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:35:41.379251311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:42.504416355Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-13T13:35:42.848938002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:35:42.857117738Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:35:43.442769604Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:35:43.446123576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41570"
ts=2026-05-13T13:35:44.096441911Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:35:44.956622427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:35:44.964048196Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=debug ts=2026-05-13T13:35:45.87997363Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-13T13:35:46.210625479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:52136"
level=warn ts=2026-05-13T13:35:46.379022752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:35:48.818791158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:35:48.825516039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:35:49.254613194Z caller=checkpoint.go:372 msg="writing series" size="722 kB"
level=warn ts=2026-05-13T13:35:51.378928122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:52.630121509Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=debug ts=2026-05-13T13:35:56.005511858Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T13:35:56.005533477Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=5
level=warn ts=2026-05-13T13:35:56.379216816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:35:59.379967807Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T13:35:59.645103715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:35:59.655445661Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T13:36:00.397733043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:44850"
level=info ts=2026-05-13T13:36:00.483478661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:36:00.489161494Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:36:01.378879961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:02.754962728Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-13T13:36:04.588070767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:36:04.591676181Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:36:05.951243952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:36:06.132451194Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:36:06.13246976Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=warn ts=2026-05-13T13:36:06.378657425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:09.50456448Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=debug ts=2026-05-13T13:36:11.210396118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:36:11.359713487Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:36:11.359792158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:36:11.379195419Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:36:11.379220447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:36:11.379230166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:36:11.379272769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:12.879548294Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
ts=2026-05-13T13:36:13.447508877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:53546"
ts=2026-05-13T13:36:14.097981586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:36:16.254739751Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T13:36:16.378852427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:19.629796106Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T13:36:21.3792306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:23.004672401Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-13T13:36:26.378720751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:26.380751559Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T13:36:26.380769924Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=6
level=info ts=2026-05-13T13:36:29.579893837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:36:29.587398191Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:36:29.755054273Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-13T13:36:31.378994799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:33.129755398Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=info ts=2026-05-13T13:36:36.149993304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:36:36.157669508Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:36:36.378438364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:36.504821844Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T13:36:37.853311476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:36:37.858211543Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:36:37.85823161Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:36:37.858254848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:36:37.863116648Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:36:39.701169183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:36:39.879903972Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-13T13:36:40.6129542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:36:40.619320261Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:36:41.378952906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:36:41.378977569Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:36:41.378987294Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:36:41.379018273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:43.254443582Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
ts=2026-05-13T13:36:43.449132121Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:39588"
ts=2026-05-13T13:36:44.099371988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T13:36:46.212430491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54504"
level=warn ts=2026-05-13T13:36:46.379191166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:46.629512175Z caller=checkpoint.go:372 msg="writing series" size="582 kB"
level=debug ts=2026-05-13T13:36:46.629534628Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=debug ts=2026-05-13T13:36:50.061199987Z caller=checkpoint.go:372 msg="writing series" size="46 MB"
level=debug ts=2026-05-13T13:36:50.061226194Z caller=checkpoint.go:384 msg="flushing series" totalSize="46 MB" series=1
level=info ts=2026-05-13T13:36:50.346895887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:36:50.352072629Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:36:50.429230068Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:36:51.37866205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:53.380052529Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-13T13:36:55.156011825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:36:55.162136164Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:36:56.379370346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:36:56.75508778Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:37:00.130093422Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
ts=2026-05-13T13:37:00.400670591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54440"
level=warn ts=2026-05-13T13:37:01.379377014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:03.505150064Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T13:37:03.730775866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:37:03.737620282Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:37:06.232667803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:37:06.378515063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:06.879118317Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:37:10.258123497Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-13T13:37:10.258144482Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=6
level=info ts=2026-05-13T13:37:11.359704603Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:37:11.359719382Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:37:11.379113181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:37:11.379132975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:37:11.379148736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:37:11.379155348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:37:11.527757812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:37:11.5353885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:37:13.63006677Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
ts=2026-05-13T13:37:14.100587041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T13:37:16.379034317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:17.00457726Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T13:37:19.626659879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:37:19.632988665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:37:20.379360706Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-13T13:37:21.378656436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:23.7557712Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=info ts=2026-05-13T13:37:26.072093933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:37:26.08016852Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=warn ts=2026-05-13T13:37:26.379097504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:27.129960068Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T13:37:27.129981334Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=5
level=debug ts=2026-05-13T13:37:30.514234629Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-13T13:37:30.514269536Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=1
level=warn ts=2026-05-13T13:37:31.37911662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:33.879779564Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:37:36.378854194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:37:36.531194341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:37:36.540762473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:37:37.254386264Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-13T13:37:40.154123216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:37:40.631259382Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T13:37:40.63128626Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=info ts=2026-05-13T13:37:41.37880545Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:37:41.378832222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:37:41.378848504Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:37:41.378841574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:37:43.451281745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:32768"
level=debug ts=2026-05-13T13:37:44.004367482Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
ts=2026-05-13T13:37:44.101809229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:37:45.373938035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:37:45.376030633Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T13:37:46.378424888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:47.380018625Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T13:37:50.754741942Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-13T13:37:51.378674152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:37:53.319302273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:37:53.326636564Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:37:53.671795874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:37:53.677221207Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:37:54.129369315Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T13:37:56.3789928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:37:57.505248834Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-13T13:38:00.879380429Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T13:38:01.379148739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:38:01.836309616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:38:01.840240423Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:38:04.255163228Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T13:38:06.378848838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:38:07.629255486Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=debug ts=2026-05-13T13:38:08.126736006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:38:11.004800229Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:38:11.359714235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:38:11.359863416Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:38:11.379203788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:38:11.379222675Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:38:11.379230335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:38:11.379227402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:38:13.452599241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50566"
ts=2026-05-13T13:38:14.103720503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:38:14.379900978Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
ts=2026-05-13T13:38:16.216399225Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36982"
level=warn ts=2026-05-13T13:38:16.379066757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:38:16.54538281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:38:16.551983167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:38:17.755008135Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T13:38:20.697805223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a2f56c5ccb0cd46 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:38:20.705307452Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a2f56c5ccb0cd46/19e21688683%3A19e217699bd%3A216a9f6c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-lxqk\"}"
level=error ts=2026-05-13T13:38:20.705327332Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:38:20.705349869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:38:20.709160093Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:38:21.129487188Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-13T13:38:21.379216134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:38:24.504769974Z caller=checkpoint.go:372 msg="writing series" size="728 kB"
level=debug ts=2026-05-13T13:38:24.504791117Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=13
level=warn ts=2026-05-13T13:38:26.37924382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:38:27.879821622Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-13T13:38:29.035787354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:38:31.254383239Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=warn ts=2026-05-13T13:38:31.379019774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:38:32.120508887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:38:32.126118768Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:38:34.6304533Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=info ts=2026-05-13T13:38:35.702095953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:38:35.709449186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:38:36.378724391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:38:38.005216362Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-13T13:38:39.705389695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:38:41.379346409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:38:41.379343511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:38:41.379377649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:38:41.379387706Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:38:41.379381213Z caller=checkpoint.go:384 msg="flushing series" totalSize="843 kB" series=4
level=info ts=2026-05-13T13:38:41.383395371Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-13T13:38:41.384499264Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005450136s
level=info ts=2026-05-13T13:38:43.989497161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:38:43.996543933Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:38:44.105039082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:38:44.37872648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:38:44.385463495Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:38:46.378869037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:38:51.378839505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:38:56.378697748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:38:59.645244992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:38:59.65201724Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:39:00.405825745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56772"
level=warn ts=2026-05-13T13:39:01.378701052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:03.920819918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:39:04.374295319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:39:04.380837087Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:39:05.699484864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:39:05.949151034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:39:05.953312461Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:39:06.378609224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:11.359220282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:39:11.360273616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:39:11.378638799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:39:11.378659816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:39:11.378667796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:39:11.378659691Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T13:39:11.378693127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:39:11.378767002Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-13T13:39:11.378775279Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:39:11.378790875Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.982µs success=true
level=debug ts=2026-05-13T13:39:11.378960896Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-13T13:39:13.455712658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34996"
ts=2026-05-13T13:39:14.106467609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:39:14.632679661Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=warn ts=2026-05-13T13:39:16.378666477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:17.886156635Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-05-13T13:39:21.140368348Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:39:21.140396548Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-13T13:39:21.378629452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:24.391109987Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-13T13:39:26.379204816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:27.64520529Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=debug ts=2026-05-13T13:39:30.897482284Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:39:31.378578887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:39:32.355292125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:39:32.364232556Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:39:33.157351196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:39:33.163931444Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:39:34.150516819Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-13T13:39:34.608116888Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:39:36.379217709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:37.403911184Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T13:39:40.657036793Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-13T13:39:41.379283834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:39:41.379315622Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:39:41.379325651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:39:41.379341552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:39:42.826957282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:39:42.835283063Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:39:43.457006553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:37540"
level=debug ts=2026-05-13T13:39:43.910147178Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-13T13:39:44.107606675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:39:46.207225905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:39:46.220550053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49418"
level=warn ts=2026-05-13T13:39:46.379372426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:47.162811292Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=debug ts=2026-05-13T13:39:50.415574821Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:39:51.378483882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:53.668527279Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T13:39:55.416867346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:39:55.422633083Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:39:55.422653987Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:39:55.42267615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:39:55.426012066Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:39:55.586547773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:39:55.590806542Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:39:56.378870636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:39:56.921736999Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-13T13:39:56.921758453Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-05-13T13:40:00.175079354Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=debug ts=2026-05-13T13:40:01.080876542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:40:01.378703347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:40:03.139256613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:40:03.145017602Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:40:03.42811319Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-13T13:40:06.379022508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:06.68124229Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-13T13:40:09.372592825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:40:09.376482262Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:40:09.934669991Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:40:09.934692701Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T13:40:11.35965113Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:40:11.36003977Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=warn ts=2026-05-13T13:40:11.3784071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:40:11.378420787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:40:11.378445086Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:40:11.378463569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:40:13.186804342Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:40:13.679204462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:40:13.687844092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:40:14.109300675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T13:40:16.378855103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:16.440071584Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=debug ts=2026-05-13T13:40:19.692291856Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T13:40:19.727393594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:40:19.734240962Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:40:19.73425881Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:40:19.734281975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:40:19.741141274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:40:21.3785507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:22.929867515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:40:22.94633843Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=debug ts=2026-05-13T13:40:26.198312071Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T13:40:26.378776152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:27.992232058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:40:29.451543069Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=warn ts=2026-05-13T13:40:31.378403699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:32.705247072Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T13:40:35.958122397Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T13:40:36.378925627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:40:38.34364389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:40:38.349950897Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:40:39.211364908Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T13:40:39.211390327Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=info ts=2026-05-13T13:40:39.300050471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:40:39.306427206Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:40:41.37878453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:40:41.378810296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:40:41.378820467Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:40:41.378810797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:40:41.890146621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:40:41.894819421Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:40:42.464706465Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T13:40:42.464727642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-13T13:40:43.645894205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:40:43.651264139Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-13T13:40:44.110516848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:40:45.2030269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:40:45.208219787Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:40:45.716763537Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-13T13:40:46.379208903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:48.970326277Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-13T13:40:51.378388055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:52.222900367Z caller=checkpoint.go:372 msg="writing series" size="853 kB"
level=debug ts=2026-05-13T13:40:55.476154307Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=warn ts=2026-05-13T13:40:56.379346825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:40:58.729082661Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T13:40:58.729104446Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=5
level=warn ts=2026-05-13T13:41:01.378401514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:01.981782287Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-13T13:41:02.228673249Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:41:04.587277385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:41:04.589014417Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:41:05.234826517Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-13T13:41:06.379098616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:41:08.379371762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:41:08.386259104Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:41:08.386282775Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:41:08.386310869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:41:08.392859551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:41:08.493365843Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:41:08.493389655Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=debug ts=2026-05-13T13:41:11.359114738Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:41:11.360165591Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:41:11.378456348Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:41:11.37845471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:11.378471209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:41:11.378479394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:41:11.741275725Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-13T13:41:14.111728621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:41:14.147413842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:41:14.156952726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:41:14.994116208Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
ts=2026-05-13T13:41:16.225235286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36580"
level=warn ts=2026-05-13T13:41:16.378746001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:18.247436937Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T13:41:19.03863293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:41:19.04593264Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:41:21.379300061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:21.499528173Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T13:41:24.753486494Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T13:41:24.950441013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:41:24.957001842Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:41:26.379390386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:28.007003143Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-13T13:41:28.00702619Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=debug ts=2026-05-13T13:41:31.259336229Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=warn ts=2026-05-13T13:41:31.378875099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:41:34.370921142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:41:34.375870889Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:41:34.512368203Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=info ts=2026-05-13T13:41:35.997632547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:41:36.002250407Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:41:36.379453937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:36.729708873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:41:37.764737912Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T13:41:41.018342411Z caller=checkpoint.go:372 msg="writing series" size="303 kB"
level=debug ts=2026-05-13T13:41:41.018363763Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T13:41:41.37929049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:41:41.379317503Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:41:41.37932836Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:41:41.37932049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:41:41.702905213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:41:41.708395968Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:41:43.014740924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:41:43.020913204Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:41:43.462980989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52692"
ts=2026-05-13T13:41:44.112633557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:41:44.270853738Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-13T13:41:46.378542473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:47.524799438Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-13T13:41:50.83126012Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-13T13:41:50.831286028Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=3
level=warn ts=2026-05-13T13:41:51.378560715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:52.654079567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:41:54.030494715Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T13:41:56.379133507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:41:57.283692112Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
ts=2026-05-13T13:42:00.414481958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:39764"
level=debug ts=2026-05-13T13:42:00.5363906Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-13T13:42:01.378842631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:03.789280984Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T13:42:03.963830529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:42:03.972247922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:42:05.507440901Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:42:05.507494612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:42:05.514686726Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T13:42:06.378649167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:07.041895408Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:42:10.298049924Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-13T13:42:10.298071146Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=6
level=debug ts=2026-05-13T13:42:11.35950655Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:42:11.360542408Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:42:11.378905973Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:42:11.378916926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:11.378924613Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:42:11.378935108Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:42:13.548804538Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
ts=2026-05-13T13:42:14.114435622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T13:42:16.378979373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:16.801192608Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T13:42:17.436099626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:42:17.444498181Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:42:17.444532563Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:42:17.444563106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:42:17.450219659Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:42:18.413148567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:42:18.420713039Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:42:20.054321339Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-13T13:42:21.379325862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:23.307547945Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:42:23.307570048Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T13:42:26.379402078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:26.561296194Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T13:42:26.561319574Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-13T13:42:29.829317961Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-13T13:42:29.829355382Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
ts=2026-05-13T13:42:30.416444232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:49694"
level=debug ts=2026-05-13T13:42:30.85046297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:42:31.222397629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:42:31.231468494Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:42:31.324850679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:42:31.330094074Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:42:31.359216531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:42:31.362526274Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T13:42:31.378712716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:42:32.377853643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:42:32.383301273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:42:33.066259858Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-13T13:42:34.604021197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:42:34.61105399Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:42:35.530775303Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:42:36.319127215Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T13:42:36.378739166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:39.57397914Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T13:42:39.574008744Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=3
level=info ts=2026-05-13T13:42:41.378701825Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:42:41.378728713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:42:41.378725943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:42:41.378737943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:42:42.826136043Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
ts=2026-05-13T13:42:44.116333846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:42:45.339480236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:42:45.348965782Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:42:46.07813959Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-13T13:42:46.230136536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:45568"
level=warn ts=2026-05-13T13:42:46.379130685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:49.331030267Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:42:51.378868934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:52.584761026Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-13T13:42:55.83745297Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T13:42:56.379350009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:42:59.09061812Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-13T13:43:00.610186673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:43:00.619208427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:43:01.378711703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:02.343349788Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-13T13:43:02.816145319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:43:05.399357789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:43:05.405531941Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:43:05.596227246Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T13:43:06.379239483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:08.849728405Z caller=checkpoint.go:372 msg="writing series" size="810 B"
level=info ts=2026-05-13T13:43:11.359621709Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:43:11.359635259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:43:11.379039202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:43:11.379035337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:11.379057597Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:43:11.379065013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:43:12.10232451Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-13T13:43:12.24673697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:43:12.250673072Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:43:12.657853824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T13:43:14.117918948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:43:15.355156879Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-13T13:43:15.355205119Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-05-13T13:43:16.231314507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53266"
level=warn ts=2026-05-13T13:43:16.378861618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:18.607985818Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T13:43:19.533556374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:43:19.540522247Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:43:21.379195531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:21.861738287Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-13T13:43:25.114745299Z caller=checkpoint.go:372 msg="writing series" size="677 kB"
level=warn ts=2026-05-13T13:43:26.379222202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:43:26.41133526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:43:26.416727588Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:43:26.416745892Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:43:26.416769029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:43:26.423026668Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:43:28.367377529Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:43:29.427368757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:43:29.431382866Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:43:30.418325588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:41814"
level=warn ts=2026-05-13T13:43:31.379381621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:31.620616142Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=debug ts=2026-05-13T13:43:32.798407641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:43:34.873446629Z caller=checkpoint.go:372 msg="writing series" size="494 kB"
level=debug ts=2026-05-13T13:43:34.873468387Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-05-13T13:43:36.378916195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:43:38.126836361Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:43:39.484635519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:43:39.492140336Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:43:41.378987877Z caller=checkpoint.go:384 msg="flushing series" totalSize="248 kB" series=1
level=info ts=2026-05-13T13:43:41.379017971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:43:41.379035427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:43:41.37904559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:43:41.379048786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:43:41.384324087Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-13T13:43:41.385704895Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006786051s
level=info ts=2026-05-13T13:43:42.719468587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:43:42.725905181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T13:43:44.119069925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:43:45.065438246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:43:45.070798421Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T13:43:46.379392185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:43:48.648630847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:43:48.653670754Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:43:50.731583959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:43:51.378967211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:43:56.379409997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T13:44:00.419232897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:36948"
level=warn ts=2026-05-13T13:44:01.379299728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:03.021959278Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:44:04.622442818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:44:04.630133418Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:44:05.719993441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:44:05.724595403Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:44:06.379304528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:44:11.359711791Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:44:11.359761959Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:44:11.379184951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:44:11.37920868Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:44:11.379206687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:44:11.379216423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:44:11.379210193Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:44:11.379342701Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-13T13:44:11.379351119Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:44:11.379370008Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.876µs success=true
level=debug ts=2026-05-13T13:44:11.379547005Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-13T13:44:14.120670527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:44:14.525329355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:44:14.531848565Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:44:14.633590988Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
ts=2026-05-13T13:44:16.234269936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48390"
level=warn ts=2026-05-13T13:44:16.3787532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:17.886594002Z caller=checkpoint.go:372 msg="writing series" size="458 kB"
level=info ts=2026-05-13T13:44:20.308112823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:44:20.313113684Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:44:21.139443894Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-13T13:44:21.139464777Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-13T13:44:21.378625748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:24.392304166Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-13T13:44:26.378618382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:27.644887039Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-13T13:44:28.620539434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:44:28.627330413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:44:29.020929284Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:44:29.022353015Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T13:44:30.89765392Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T13:44:31.379165768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:44:31.961083383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:44:31.965659725Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:44:34.151646877Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-13T13:44:36.155936733Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:44:36.379109331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:37.403896654Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T13:44:40.657528632Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-13T13:44:41.37832167Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:44:41.378353645Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:44:41.378361393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:44:41.378370996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:43.910365764Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-05-13T13:44:44.102831456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:44:44.112048961Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:44:44.12231219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T13:44:46.379404787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:47.164068329Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-13T13:44:47.164088903Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-13T13:44:48.054185695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:44:48.062078475Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:44:50.416718089Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:44:51.37913129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:44:51.535874609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:44:51.54376895Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T13:44:51.543786689Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:44:51.54380849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:44:51.547420543Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:44:53.668997406Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T13:44:56.378602626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:44:56.922387482Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-13T13:45:00.175568354Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-13T13:45:01.378417139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:45:03.324801143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:45:03.331614395Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:45:03.428186202Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T13:45:04.641451264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:45:04.646744739Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:45:05.308307222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:45:05.312471423Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T13:45:06.379189558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:06.681466982Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-13T13:45:09.577437682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:45:09.577461677Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T13:45:09.581022196Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:45:09.877778869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="48 kB" avg_uncomp="48 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:45:09.880871871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-13T13:45:09.934513461Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:45:09.934526136Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=info ts=2026-05-13T13:45:10.177901578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="30 kB" avg_uncomp="30 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:45:10.181065348Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:45:10.477911999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:45:10.480875508Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:45:10.777762696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:45:10.781155593Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:45:11.359569446Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:45:11.360624491Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:45:11.360824794Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T13:45:11.360870764Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976310 n=1
level=debug ts=2026-05-13T13:45:11.360922248Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976310 n=1
level=info ts=2026-05-13T13:45:11.378970636Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:45:11.3790049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:45:11.379002494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:45:11.379012745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:45:13.187765756Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-13T13:45:14.12336949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=warn ts=2026-05-13T13:45:16.379242573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:16.44040681Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T13:45:16.662758532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:45:16.668697179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:45:16.86396656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:45:16.868599397Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:45:19.693090261Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-13T13:45:20.756171484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:45:20.759342615Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:45:21.37866514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:22.946752727Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=info ts=2026-05-13T13:45:24.621291069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:45:24.624298634Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:45:24.708478201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:45:24.711224861Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:45:26.19908181Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-13T13:45:26.378518669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:45:26.714214076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:45:26.717150324Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-13T13:45:26.81389631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:45:27.278687668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:45:27.281705136Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:45:29.452481027Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-13T13:45:29.452503595Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-13T13:45:30.43493201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:45:30.442331797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:45:30.595877398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:45:30.599652588Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:45:31.378898137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:32.705404367Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T13:45:35.958543329Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T13:45:36.378466268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:39.211433294Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=info ts=2026-05-13T13:45:41.378508023Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:45:41.378533162Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:45:41.378541994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:45:41.378563523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:42.465521512Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T13:45:42.465544047Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-13T13:45:43.010134136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=debug ts=2026-05-13T13:45:43.010192921Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T13:45:43.015514404Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:45:43.432703329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:45:43.438515585Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:45:44.124687097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:45:45.717433859Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-13T13:45:46.379232244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:45:47.466600802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:45:47.469774203Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:45:48.969963731Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T13:45:49.875416534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:45:49.880886246Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:45:51.229653351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:45:51.234502347Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:45:51.3218539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:45:51.324233651Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:45:51.378846792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:45:52.22375511Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T13:45:52.223775503Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-13T13:45:53.752649115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:45:53.756412612Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:45:55.285444579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:45:55.288466956Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:45:55.476565524Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=warn ts=2026-05-13T13:45:56.37862835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:45:58.64807636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:45:58.652684661Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:45:58.730607487Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T13:45:58.730621824Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T13:46:01.379141967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:01.982487366Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T13:46:04.112368255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:46:04.116459763Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-13T13:46:05.136187389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:46:05.139253345Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:46:05.235670442Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-13T13:46:05.363288147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:46:05.369070204Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:46:06.378800793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:08.49202755Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:46:08.492050019Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=debug ts=2026-05-13T13:46:09.164839758Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:46:09.164891244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:46:09.16754347Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:46:09.832036909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:46:11.359406002Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:46:11.360451505Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:46:11.378818295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:46:11.378825062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:11.378836409Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:46:11.378843985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:46:11.742003477Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-13T13:46:14.125937841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:46:14.994382298Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
level=info ts=2026-05-13T13:46:15.424391585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:46:15.432110685Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:46:16.37883663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:18.247692859Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T13:46:20.76425823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:46:20.768655863Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:46:21.378384078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:21.500720433Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T13:46:23.803432277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:46:23.809986597Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:46:24.420636438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:46:24.423804276Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:46:24.753713309Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T13:46:26.379215314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:28.008999459Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T13:46:28.009022097Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=6
level=debug ts=2026-05-13T13:46:31.260159739Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-13T13:46:31.378986792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:46:32.146974415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:46:32.150323169Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:46:34.513148795Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=info ts=2026-05-13T13:46:34.775789975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:46:34.778862606Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:46:35.451837433Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:46:35.848805701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:46:35.851981685Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:46:36.379126238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:46:36.881358608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:46:36.884753697Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:46:36.884772183Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:46:36.884792964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:46:36.887677222Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:46:37.765979352Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T13:46:40.521570245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:46:40.526752388Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:46:41.018249256Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-13T13:46:41.378356895Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:46:41.37838395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:46:41.378391628Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:46:41.378388486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:46:42.729715856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:46:42.734845183Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T13:46:44.126933891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:46:44.272232705Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
ts=2026-05-13T13:46:46.240712172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59170"
level=warn ts=2026-05-13T13:46:46.37857871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:47.524949211Z caller=checkpoint.go:372 msg="writing series" size="543 kB"
level=debug ts=2026-05-13T13:46:47.524973082Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-13T13:46:48.258811345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:46:48.26189949Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:46:48.725085334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:46:48.728749998Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:46:50.042283262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:46:50.045347221Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:46:50.832580848Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T13:46:50.832605249Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=1
level=warn ts=2026-05-13T13:46:51.378399164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:54.030523153Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-13T13:46:55.89962177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:46:55.908737004Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:46:55.908764797Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:46:55.908797826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:46:55.915054556Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:46:56.379381167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:46:57.284444022Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=info ts=2026-05-13T13:46:59.70910481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:46:59.712441785Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:47:00.536671029Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-13T13:47:01.379015194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:02.256646058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:47:03.774227745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:47:03.777052714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-13T13:47:03.790126026Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-13T13:47:04.174106693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:47:05.754598609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:47:05.759150745Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:47:06.341612674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:47:06.348815912Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:47:06.34883716Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:47:06.348861689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:47:06.355468624Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:47:06.378662025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:47:06.763427298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:47:06.770011712Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:47:07.04282823Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:47:10.297953443Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-13T13:47:10.297976271Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=6
level=info ts=2026-05-13T13:47:10.501890635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:47:10.507514652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:47:11.359151938Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:47:11.360203106Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:47:11.378513616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:47:11.37851952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:11.378535468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:47:11.378543726Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:47:13.54834155Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
ts=2026-05-13T13:47:14.127964568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:47:15.601865847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:47:15.604669185Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T13:47:16.379389417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:16.802016348Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-13T13:47:20.05457658Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-13T13:47:21.378895829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:47:22.794165826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:47:22.79950915Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:47:23.308558914Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:47:23.308585184Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T13:47:24.337675041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:47:24.341733867Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:47:25.442928583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:47:25.450722948Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:47:26.378615234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:26.561236762Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T13:47:26.561264364Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-13T13:47:28.64109592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:47:28.645997412Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:47:28.866618066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:47:28.874278288Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:47:29.821611932Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-13T13:47:29.821636735Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-05-13T13:47:31.378544587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:33.06642044Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-13T13:47:36.320213111Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T13:47:36.378963526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:47:37.025756846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:47:37.033139698Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:47:37.136533101Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:47:37.994055534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:47:37.997000526Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:47:39.574329489Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T13:47:39.574352631Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=3
level=info ts=2026-05-13T13:47:41.378336149Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:47:41.378371587Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:47:41.378382301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:47:41.378389588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:47:41.803133707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:47:41.808738249Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:47:42.82652374Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
ts=2026-05-13T13:47:44.129236565Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:47:44.341594476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:47:44.344607224Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:47:46.078476779Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T13:47:46.3790681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:49.332000351Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-13T13:47:49.950644901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:47:49.953793344Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:47:51.378919538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:47:52.585387098Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-13T13:47:55.802771096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:47:55.837832203Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-13T13:47:56.378767592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:47:57.101704091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:47:57.108845225Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:47:57.309393904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:47:57.313421129Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:47:59.091649777Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-13T13:47:59.541862693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:47:59.549749062Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-13T13:48:00.043742829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:48:00.049337034Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:48:01.378714383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:48:02.34361874Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-13T13:48:03.100733169Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:48:03.100737304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="189 kB" avg_uncomp="189 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:48:03.1040665Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:48:03.401137197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="344 B" avg_uncomp="344 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:48:03.404053337Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:48:05.597105801Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-13T13:48:06.100577552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:48:06.103658083Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T13:48:06.379217142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:48:08.850511722Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-13T13:48:11.024232552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:48:11.028500298Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:48:11.359769075Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:48:11.359777285Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:48:11.37920355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:48:11.379204729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:48:11.379219384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:48:11.379227147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:48:12.102807968Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
ts=2026-05-13T13:48:14.130765865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:48:15.35629164Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=debug ts=2026-05-13T13:48:15.35631345Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-13T13:48:16.378715321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:17.532648024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:48:17.535855996Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:48:17.845749193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:48:17.84877581Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T13:48:18.608943418Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T13:48:19.130206061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:48:19.133923083Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T13:48:21.37908362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:21.677074577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:48:21.680758517Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:48:21.862568633Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-13T13:48:24.794003315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:48:24.797831614Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:48:25.116240821Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=warn ts=2026-05-13T13:48:26.378980443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:27.736520281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:48:27.741769791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:48:28.367824957Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-13T13:48:31.378519575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:48:31.533336393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:48:31.62067259Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=info ts=2026-05-13T13:48:34.398649845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:48:34.403149832Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:48:34.874771493Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=debug ts=2026-05-13T13:48:34.874794017Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-13T13:48:35.76174637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:48:35.76899697Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:48:36.379384634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:37.32437586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:48:37.329582818Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:48:37.329606137Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:48:37.329630321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:48:37.334452244Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:48:38.126920893Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:48:38.50103254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:48:38.508108781Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:48:41.378505606Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:48:41.378529213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:48:41.378544081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:48:41.378523822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:48:41.379603286Z caller=checkpoint.go:384 msg="flushing series" totalSize="248 kB" series=1
level=info ts=2026-05-13T13:48:41.384360048Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-13T13:48:41.386009396Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006497596s
level=debug ts=2026-05-13T13:48:41.80807236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:48:41.809689331Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T13:48:43.055899654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:48:43.058903235Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T13:48:44.131967589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:48:45.464607078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:48:45.467535411Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-13T13:48:46.378742292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:47.409341427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:48:47.414611329Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:48:48.806150526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:48:48.808885735Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T13:48:49.612170355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:48:49.614919884Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T13:48:49.697999769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:48:49.703191553Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:48:49.703208217Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:48:49.703229459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:48:49.707728449Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:48:51.378479158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:54.419594204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:48:54.425733274Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T13:48:56.379450667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:48:56.570997827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:48:56.574603394Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-13T13:49:00.433726926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:53314"
level=debug ts=2026-05-13T13:49:00.631659672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:49:01.37855606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:49:04.957438544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:49:04.965092753Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:49:06.032765058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:49:06.03613777Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T13:49:06.378771478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:49:08.305296276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:49:08.309460795Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:49:11.359707644Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:49:11.359916742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:49:11.37925734Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:49:11.379286491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:49:11.379304896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:49:11.379312916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:49:11.37932702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:49:11.379373183Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-13T13:49:11.379380241Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:49:11.379398044Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.103µs success=true
level=debug ts=2026-05-13T13:49:11.379569787Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T13:49:13.267366011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:49:13.270750359Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T13:49:14.133375264Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:49:14.63299188Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-13T13:49:15.011735299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:49:15.014552276Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T13:49:16.246644418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:40748"
level=warn ts=2026-05-13T13:49:16.378882283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:17.886041418Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=debug ts=2026-05-13T13:49:21.139690842Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-13T13:49:21.379047879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:49:22.352656327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:49:22.355823069Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T13:49:22.759058398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:49:22.764854142Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:49:23.076674891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:49:23.082546056Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:49:24.391867013Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-13T13:49:24.391887291Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-13T13:49:25.394524969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:49:26.21796895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:49:26.220666796Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-13T13:49:26.378541315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:27.645659599Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-13T13:49:29.618979762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:49:29.624451332Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T13:49:29.70364968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:49:29.706055204Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T13:49:30.435425885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59370"
level=debug ts=2026-05-13T13:49:30.898322583Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T13:49:31.378792445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:32.047899721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:49:32.19021469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:49:32.192945067Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:49:33.274198088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:49:33.280102067Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:49:34.151539042Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-13T13:49:36.379253074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:37.404449553Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T13:49:40.657385182Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-13T13:49:41.378765818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:49:41.378764806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:41.378786347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:49:41.378794957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:49:43.910716537Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-13T13:49:44.135460842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:49:44.298429955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:49:44.305275882Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:49:44.956154654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:49:44.961554982Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:49:46.24749939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43504"
level=warn ts=2026-05-13T13:49:46.37894943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:49:47.00410212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:49:47.007458985Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:49:47.163063179Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-13T13:49:50.035324339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:49:50.038256247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:49:50.41649814Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:49:51.379413932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:53.669080866Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T13:49:55.31543268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:49:55.318014739Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T13:49:56.379369069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:49:56.921869306Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=debug ts=2026-05-13T13:49:56.921888016Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T13:49:59.169583807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:49:59.173483508Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:50:00.175314826Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-13T13:50:01.379123058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:01.80205048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:50:03.428269677Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T13:50:05.198844023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:50:05.201933372Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:50:05.451208302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:50:05.455950296Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:50:05.455966923Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:50:05.455990052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:50:05.460510362Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:50:05.890770212Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:50:06.37888982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:06.681592708Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T13:50:09.934570029Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:50:09.934594208Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T13:50:11.359639978Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:50:11.359937298Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:50:11.377205663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:50:11.378536658Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:50:11.378558889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:50:11.378567535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:50:11.378567361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-13T13:50:11.379858128Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:50:13.187324464Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T13:50:13.630734391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:50:13.635828248Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T13:50:14.13723059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:50:15.222919403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:50:15.225491298Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-13T13:50:16.378578596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:16.440741919Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T13:50:16.858869783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:50:16.86209313Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T13:50:17.921791199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:50:17.926547816Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T13:50:19.612373609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:50:19.618380013Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:50:19.693683057Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-13T13:50:20.484063267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:50:20.486824958Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T13:50:20.953583425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:50:20.960307225Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:50:21.379405265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:22.946677746Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=info ts=2026-05-13T13:50:24.583898257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:50:24.589396577Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:50:26.199766055Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-13T13:50:26.378982313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:50:27.096544815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:50:27.100973383Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:50:29.4529576Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=debug ts=2026-05-13T13:50:29.45298874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T13:50:31.378691564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:32.705189364Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T13:50:35.633604249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:50:35.636567252Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:50:35.94636552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:50:35.949397095Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:50:35.958582756Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T13:50:36.379225763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:39.211601311Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T13:50:39.705578913Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:50:40.629197057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:50:40.634459216Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:50:41.015526214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:50:41.018921565Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T13:50:41.378335637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:50:41.378359867Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:50:41.378367582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:50:41.378412449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:42.465970798Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T13:50:42.465991604Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
ts=2026-05-13T13:50:43.487613036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:35108"
ts=2026-05-13T13:50:44.139017124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:50:44.31112904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:50:44.31649031Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:50:45.245076706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:50:45.248340168Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:50:45.717835235Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
ts=2026-05-13T13:50:46.250427316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38534"
level=warn ts=2026-05-13T13:50:46.379062189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:48.970907197Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T13:50:49.655515796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:50:49.660335126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:50:50.805664072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:50:51.378764616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:52.223715284Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=debug ts=2026-05-13T13:50:55.476378144Z caller=checkpoint.go:372 msg="writing series" size="992 B"
level=warn ts=2026-05-13T13:50:56.379213474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:50:58.730459361Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T13:50:58.730479729Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=5
level=warn ts=2026-05-13T13:51:01.378884594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:01.592410295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:51:01.598883577Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T13:51:01.982445086Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T13:51:02.103283794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:51:02.106844797Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T13:51:02.373807257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:51:02.377081839Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:51:02.474320349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:51:02.477459351Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T13:51:03.723458063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:51:03.728739224Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:51:05.235823902Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-13T13:51:05.544756572Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:51:06.378658654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:06.407811865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:51:06.410464111Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-13T13:51:07.383219244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:51:07.389492461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:51:08.492525715Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:51:08.49255214Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=debug ts=2026-05-13T13:51:11.359602335Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:51:11.359625395Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:51:11.379011218Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:51:11.379007507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:51:11.379029964Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:51:11.379039304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:51:11.742067842Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-13T13:51:14.140261804Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:51:14.995007178Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
ts=2026-05-13T13:51:16.251362316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:37108"
level=warn ts=2026-05-13T13:51:16.378966081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:16.862060495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:51:16.866672378Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:51:18.247110242Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T13:51:19.667372113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.4 MB" total_uncomp="84 MB" avg_uncomp="21 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:51:19.673419881Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:51:20.269876611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:51:20.272688798Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:51:20.272710286Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:51:20.272731479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:51:20.274819533Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:51:20.757494924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:51:20.760998514Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T13:51:20.76101722Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:51:20.761040124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:51:20.764485355Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:51:21.030132985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:51:21.035058303Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:51:21.378952683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:51:21.500294018Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T13:51:24.75420713Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-13T13:51:26.378722796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:51:27.452169041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:51:28.008649335Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-13T13:51:28.008671362Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=info ts=2026-05-13T13:51:30.502534907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:51:30.504992953Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:51:30.505009188Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:51:30.505029205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:51:30.507089684Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:51:31.259404618Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-13T13:51:31.378416688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:32.15011225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:51:32.153069514Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:51:34.512802382Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=info ts=2026-05-13T13:51:35.168168263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:51:35.170830542Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:51:36.379108175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:37.393218049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:51:37.396819465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:51:37.766100449Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T13:51:38.068962501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:51:38.073769435Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:51:38.231341159Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:51:41.018411032Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-13T13:51:41.378791868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:51:41.378811539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:51:41.378819832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:51:41.378824242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:43.558212053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:51:43.561385356Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T13:51:44.142039398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:51:44.271326987Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-13T13:51:46.378747051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:47.055917313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:51:47.062462032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:51:47.524975993Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=debug ts=2026-05-13T13:51:47.524999041Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-13T13:51:48.955378913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:51:48.958276551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:51:50.841654481Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T13:51:50.841683648Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=1
level=warn ts=2026-05-13T13:51:51.378854695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:53.686912334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:51:53.696519112Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:51:54.030583916Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-13T13:51:54.703428506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:51:54.706731821Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e215dcade%3A19e215e4b17%3A72928a4d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-13T13:51:54.706750474Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:51:54.706772085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="27 kB" avg_comp="27 kB" total_uncomp="288 kB" avg_uncomp="288 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:51:54.709248375Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T13:51:54.824639107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:51:54.827409316Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:51:56.297408197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:51:56.300465442Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T13:51:56.378914652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:51:56.713054581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:51:56.716012935Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:51:56.716031505Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:51:56.716050328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:51:56.718481096Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:51:57.284270211Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T13:52:00.537196449Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-13T13:52:01.37839713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:52:02.736013837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:52:02.742065749Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:52:03.79026168Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-13T13:52:06.37924973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:07.043244447Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T13:52:07.867026201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:52:07.870210805Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T13:52:08.7875736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:52:08.789272742Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T13:52:08.884497312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:52:08.888281822Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:52:09.274417037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:52:09.276999766Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:52:10.298751095Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T13:52:10.29877553Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=6
level=info ts=2026-05-13T13:52:10.73994613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:52:10.742969595Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:52:11.359711807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:52:11.359715895Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:52:11.379122307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:52:11.379139999Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:52:11.379148169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:52:11.379201264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:13.313087845Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:52:13.548967124Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-05-13T13:52:14.026476834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:52:14.029668063Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T13:52:14.029687969Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:52:14.029711221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:52:14.032113632Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-13T13:52:14.143548104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:52:15.742472956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:52:15.745628611Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:52:16.378952317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:16.802246455Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T13:52:19.360228995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:52:19.367000582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:52:20.055447269Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-13T13:52:20.434369857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:52:20.440023015Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:52:21.129910373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:52:21.132703662Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:52:21.1327219Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:52:21.132740669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:52:21.134839291Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:52:21.378386793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:23.308921849Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:52:23.308943879Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-13T13:52:26.379360139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:26.561513589Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T13:52:26.561531666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-13T13:52:29.825151822Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-13T13:52:29.825197054Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-05-13T13:52:31.37883899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:52:32.654804546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:52:32.660819839Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:52:33.06702041Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-13T13:52:34.229125763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:52:34.233743511Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T13:52:35.125283182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:52:35.128226751Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T13:52:35.221502886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:52:35.22503919Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:52:36.320242276Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T13:52:36.37935441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:52:38.143224664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:52:38.146607202Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:52:39.575002792Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T13:52:39.575023226Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-13T13:52:40.096145779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:52:40.103412666Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:52:40.1164402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:52:40.118927473Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T13:52:41.379060064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:52:41.379061945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:41.379080894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:52:41.379089777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:52:42.825660343Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=debug ts=2026-05-13T13:52:43.058232031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:52:44.144359418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:52:44.54821942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:52:44.553957267Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:52:44.722290694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:52:44.725543287Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:52:44.725560851Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:52:44.725581084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:52:44.727958494Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:52:46.079015047Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-13T13:52:46.255433526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49718"
level=warn ts=2026-05-13T13:52:46.378896671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:52:46.978105735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:52:46.9810774Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T13:52:49.332555918Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:52:51.378382235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:52:52.584887128Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:52:55.745787882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:52:55.749293268Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:52:55.837519171Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=debug ts=2026-05-13T13:52:56.369599306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:52:56.378639831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:52:57.467311492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:52:57.4703087Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:52:57.517357604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:52:57.521465331Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:52:59.091032951Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-13T13:52:59.955460194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:52:59.961027958Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T13:53:01.165433193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:53:01.168045128Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:53:01.378483127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:53:02.344616551Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-13T13:53:03.476001736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:53:03.478833946Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:53:05.596758319Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T13:53:06.378805155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:53:06.844376189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:53:06.847069528Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:53:07.157534641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:53:07.424153777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:53:07.430580801Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:53:08.850413935Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=debug ts=2026-05-13T13:53:11.359260877Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:53:11.360301794Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T13:53:11.378661572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:53:11.378670371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:53:11.378691782Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:53:11.378699125Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:53:12.103143248Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
ts=2026-05-13T13:53:14.145792805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:53:15.356089947Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-13T13:53:15.356112661Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-13T13:53:15.696951683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:53:15.702879209Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T13:53:16.378975371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:53:18.608786559Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T13:53:18.729028799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:53:18.731975936Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T13:53:21.379030118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:53:21.86173845Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-13T13:53:23.022232228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:53:23.025435233Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:53:24.806714962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:53:24.813441308Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:53:24.813461372Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:53:24.813485736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:53:24.819068612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:53:25.11548469Z caller=checkpoint.go:372 msg="writing series" size="670 kB"
level=info ts=2026-05-13T13:53:25.853023685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:53:25.855813253Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T13:53:26.378957157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:53:27.91951369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:53:27.923003731Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:53:27.92302628Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:53:27.923050761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:53:27.925912791Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:53:28.015202359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:53:28.017592187Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T13:53:28.367904261Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-13T13:53:29.43885803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:53:29.441927505Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T13:53:31.379046705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:53:31.621470493Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=debug ts=2026-05-13T13:53:32.89153471Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:53:34.87501258Z caller=checkpoint.go:372 msg="writing series" size="636 kB"
level=debug ts=2026-05-13T13:53:34.875034858Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=warn ts=2026-05-13T13:53:36.379430946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:53:38.12736801Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:53:38.407212381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:53:38.411465454Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:53:39.409369921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:53:39.413054207Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:53:41.224749982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:53:41.229715125Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T13:53:41.378860284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:53:41.378885423Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:53:41.378878788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:53:41.378894517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:53:41.379927632Z caller=checkpoint.go:384 msg="flushing series" totalSize="248 kB" series=1
level=info ts=2026-05-13T13:53:41.386527373Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-13T13:53:41.387774737Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008244407s
level=info ts=2026-05-13T13:53:43.363810665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:53:43.37078907Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:53:43.495330985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41878"
ts=2026-05-13T13:53:44.147468614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:53:46.090844736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:53:46.095962621Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T13:53:46.378805589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:53:46.596478056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:53:46.599109876Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:53:47.706670256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:53:51.37873106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:53:52.123749615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:53:52.126979298Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:53:55.062296333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:53:55.065297489Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T13:53:56.378446028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:53:56.942194564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:53:59.005090802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:53:59.008398521Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T13:54:00.446943145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:47250"
level=warn ts=2026-05-13T13:54:01.378700862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T13:54:06.379383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:06.665218267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:54:06.673404886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:54:06.673423634Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:54:06.673451111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:54:06.679133736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:54:07.648801488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:54:07.652014761Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:54:07.652034434Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:54:07.652055853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:54:07.654484758Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T13:54:08.251154254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:54:08.255678039Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T13:54:08.284744561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:54:08.287558713Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T13:54:11.359529671Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:54:11.360572486Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:54:11.378918019Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:54:11.37894681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:54:11.378962828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:54:11.378973064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:54:11.378977606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:11.379066879Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-13T13:54:11.379074099Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:54:11.379089613Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.297µs success=true
level=debug ts=2026-05-13T13:54:11.37929395Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T13:54:13.613100764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:54:13.618638442Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T13:54:14.149394551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:54:14.632763586Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-13T13:54:14.980660426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:54:14.983699797Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T13:54:15.330487163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:54:15.336957848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:54:16.378736545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:54:17.885633426Z caller=checkpoint.go:372 msg="writing series" size="494 kB"
level=info ts=2026-05-13T13:54:18.277558958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:54:18.280904401Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:54:21.139263668Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-13T13:54:21.139285997Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T13:54:21.379216798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:23.227290521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:54:23.230598723Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:54:24.392495396Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-13T13:54:24.525599432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:54:24.53260383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:54:26.37922758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:26.634703107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:54:26.637734533Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:54:26.637753459Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:54:26.637772298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:54:26.639927845Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T13:54:27.64553282Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-13T13:54:27.679669017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:54:27.682391643Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:54:29.975440317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:54:29.979605806Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T13:54:30.447805906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59040"
level=debug ts=2026-05-13T13:54:30.897388058Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T13:54:31.379344147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:54:31.466710605Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:54:34.151109318Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-13T13:54:36.379191295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:54:37.403763278Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T13:54:38.902764878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:54:38.906981641Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:54:40.657110254Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-13T13:54:41.11933895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:54:41.124805601Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-13T13:54:41.378696216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:41.378711863Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:54:41.378734729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:54:41.378749129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:54:41.532144317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:54:41.53534261Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:54:42.267401892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:54:42.272289841Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-13T13:54:43.497995612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47368"
level=debug ts=2026-05-13T13:54:43.909917918Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-13T13:54:44.15078577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:54:46.035042419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:54:46.037795778Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T13:54:46.261536867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36978"
level=warn ts=2026-05-13T13:54:46.379187571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:46.479511106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:54:46.482432162Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:54:47.162728422Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-13T13:54:47.162759308Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-13T13:54:48.122370381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:54:49.697607448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:54:49.70158141Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:54:50.415731973Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:54:51.379375223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:54:53.66859254Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T13:54:56.378740518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:54:56.765686911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:54:56.771647453Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:54:56.922320587Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-13T13:54:57.659899095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:54:57.66322732Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:55:00.175549767Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-13T13:55:00.448715605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33868"
level=info ts=2026-05-13T13:55:01.148800113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:55:01.155833903Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T13:55:01.379267608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:55:03.428237196Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-13T13:55:04.260419833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:55:06.379394721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:06.536087764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:55:06.539220854Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T13:55:06.681523592Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-13T13:55:08.829092444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:55:08.835527074Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:55:09.93460106Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T13:55:09.934623067Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=debug ts=2026-05-13T13:55:11.359576476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:55:11.360625815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:55:11.378975417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:55:11.378994238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:55:11.379001727Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:55:11.37901753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:12.926312091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:55:12.931241748Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:55:13.187207578Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-13T13:55:14.152132453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T13:55:14.16506152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:55:14.168248432Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:55:16.378609824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:55:16.439791434Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T13:55:17.115769638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:55:17.118947168Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:55:18.337286124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:55:18.341228394Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:55:19.693186197Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T13:55:21.37852184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:55:22.946316548Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=info ts=2026-05-13T13:55:23.520696346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:55:23.524485065Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:55:25.087649722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:55:25.09286726Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:55:26.199259488Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-13T13:55:26.379155269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:26.558546075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:55:26.561516005Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:55:29.452294512Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=debug ts=2026-05-13T13:55:29.452318614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-13T13:55:29.840062258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:55:29.843025837Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T13:55:31.378395324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:31.510700965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:55:31.513574625Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:55:31.694347763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:55:31.697516456Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:55:32.705576687Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T13:55:33.410636588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:55:33.415464695Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:55:33.415488219Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:55:33.415524889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:55:33.420109989Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:55:35.958227296Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T13:55:36.378634118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:55:36.454005666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:55:36.454063631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:55:36.45667516Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T13:55:39.177844827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:55:39.211260633Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=info ts=2026-05-13T13:55:41.379203041Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:55:41.379210808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:55:41.379226768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:55:41.379234393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:55:42.106430428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:55:42.113197184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T13:55:42.465368962Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-13T13:55:42.465388686Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
ts=2026-05-13T13:55:44.153301257Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T13:55:45.421386071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:55:45.425865619Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:55:45.717775304Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-13T13:55:46.379375745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:48.772758666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:55:48.777598734Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:55:48.970189597Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T13:55:51.378750008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:51.47802382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:55:51.480927572Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T13:55:52.223709024Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T13:55:52.223731694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-13T13:55:54.732937922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:55:54.737452079Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T13:55:55.042306214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:55:55.047586403Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:55:55.047605314Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:55:55.047627757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:55:55.052485752Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:55:55.475876229Z caller=checkpoint.go:372 msg="writing series" size="946 B"
level=warn ts=2026-05-13T13:55:56.37840897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:55:56.869606272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:55:56.872312226Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:55:58.72942234Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T13:55:58.72944561Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T13:55:59.960720303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:55:59.967766499Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T13:56:00.450830465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:51336"
level=info ts=2026-05-13T13:56:01.321671521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:56:01.324979923Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T13:56:01.379235627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:01.982528237Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-13T13:56:05.235283527Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-13T13:56:06.379366145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:08.49360944Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T13:56:08.493629614Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=debug ts=2026-05-13T13:56:08.582045594Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:56:10.209300936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:56:10.216094304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:56:11.359077602Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:56:11.36012998Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T13:56:11.378433177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:56:11.378457223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:56:11.378466211Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:56:11.378471895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:11.741270865Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-13T13:56:13.169223363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:56:13.17443202Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:56:13.866366706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:56:13.870491509Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:56:14.036808649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:56:14.039861744Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T13:56:14.154384317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:56:14.994372855Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-13T13:56:15.280074372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:56:15.282784736Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T13:56:16.378870234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:18.247287989Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T13:56:18.488926345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:56:18.491381951Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T13:56:19.527597314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:56:19.530457563Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:56:19.796292071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:56:19.799078151Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:56:20.474109994Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T13:56:20.475734615Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T13:56:20.526806647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:56:20.531478553Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T13:56:21.378923803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:21.500314982Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T13:56:22.446587716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:56:22.449506035Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T13:56:24.050693742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:56:24.053735935Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:56:24.753873081Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-05-13T13:56:26.378908392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:28.008301588Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T13:56:28.008323062Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=6
level=debug ts=2026-05-13T13:56:31.259812171Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-13T13:56:31.378676489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:32.772801963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:56:34.512847293Z caller=checkpoint.go:372 msg="writing series" size="564 kB"
level=info ts=2026-05-13T13:56:35.976445301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:56:35.981593075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T13:56:36.378616955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:37.765272385Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T13:56:41.018951676Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-13T13:56:41.286727034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:56:41.293463542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:56:41.37889909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:56:41.378911804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:56:41.378934301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:56:41.378944148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:56:42.807965076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:56:42.811780149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T13:56:43.503138969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:38832"
ts=2026-05-13T13:56:44.155300658Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:56:44.271170847Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T13:56:46.266160947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47900"
level=warn ts=2026-05-13T13:56:46.379260173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:56:47.044770117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:56:47.055452532Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T13:56:47.525072812Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-13T13:56:47.525093362Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-13T13:56:47.771649501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:56:47.77449023Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:56:49.303383364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:56:49.30595751Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T13:56:50.834654607Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T13:56:50.834677355Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=1
level=warn ts=2026-05-13T13:56:51.379382928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:56:53.53938807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:56:53.547531733Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T13:56:54.030815404Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-13T13:56:56.37893844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:56:57.28346532Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=info ts=2026-05-13T13:57:00.006020683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:57:00.009046312Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:57:00.536649241Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-13T13:57:00.719636351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:57:01.378914856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:02.205352147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:57:02.210825511Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T13:57:03.34066885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:57:03.343671766Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:57:03.78924856Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T13:57:05.747710083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:57:05.750737558Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T13:57:06.379412763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:06.523761085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:57:06.529164382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:57:06.529198886Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:57:06.529223358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:57:06.534294795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:57:07.042592294Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T13:57:08.50621875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:57:08.506240002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T13:57:08.509343934Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T13:57:09.713621498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:57:09.716697914Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:57:09.730756708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:57:09.734950575Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:57:10.299736997Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T13:57:10.299765244Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=6
level=info ts=2026-05-13T13:57:11.359706574Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:57:11.359920105Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:57:11.379279008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:57:11.379298435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:57:11.379309382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:57:11.37932693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:12.696905947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:57:12.700020267Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T13:57:13.548614242Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=info ts=2026-05-13T13:57:13.614711002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:57:13.617800207Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T13:57:14.156411212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T13:57:16.378876985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:16.556081952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:57:16.56113883Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:57:16.801489423Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T13:57:19.705523477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:57:19.711018346Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T13:57:20.054195598Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=warn ts=2026-05-13T13:57:21.378664837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:22.811882996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T13:57:22.814975607Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T13:57:23.308409483Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T13:57:23.308430935Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-13T13:57:25.260820681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:57:25.267459015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:57:26.379291226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:57:26.561992658Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T13:57:26.56201044Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=debug ts=2026-05-13T13:57:29.824501207Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-13T13:57:29.824523313Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
ts=2026-05-13T13:57:30.454443381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42842"
level=info ts=2026-05-13T13:57:30.803758635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:57:30.806916867Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:57:31.379019842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:57:33.06707608Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-13T13:57:34.282733737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:57:34.285727784Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T13:57:35.01269158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:57:35.016257842Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:57:36.319806213Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T13:57:36.378748103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:57:37.357714936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T13:57:37.4990112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:57:37.507608704Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:57:38.221526237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:57:38.224966474Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T13:57:39.575215018Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T13:57:39.575235631Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-13T13:57:40.091142859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:57:40.096689712Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:57:41.379016917Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:57:41.379038588Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:57:41.379046181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:57:41.379059852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:57:42.825350951Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
ts=2026-05-13T13:57:44.157951642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T13:57:44.739950262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:57:44.745068982Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:57:46.078674985Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-05-13T13:57:46.142750532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:57:46.145291199Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:57:46.145307721Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:57:46.145327672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:57:46.147526375Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T13:57:46.37855078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:46.658084316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:57:46.663202491Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:57:47.621767699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T13:57:49.331965164Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T13:57:51.378967781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:57:52.584838064Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T13:57:53.062192073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T13:57:53.064981852Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T13:57:55.837667197Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-13T13:57:56.378916068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:57:56.880299224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:57:56.883190313Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:57:58.240285645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:57:58.243124703Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:57:58.243146614Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:57:58.243168763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:57:58.245805727Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:57:58.412589324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:57:58.416012683Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:57:59.090853046Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-13T13:57:59.106747355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:57:59.109561719Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T13:58:01.378707755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:01.571624833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:58:01.574233433Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T13:58:02.343388886Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-13T13:58:05.547228414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:58:05.552765748Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T13:58:05.552781603Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:58:05.552801983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:58:05.556451813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T13:58:05.597250544Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-13T13:58:06.310059043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:58:06.312862103Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:58:06.37916589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:06.64941924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:58:06.651978978Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T13:58:06.758301296Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:58:08.849802027Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-13T13:58:08.861925358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:58:08.869073633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:58:11.359633111Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:58:11.35964892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:58:11.379050656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:58:11.379073715Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:58:11.379080817Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:58:11.379074453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:58:12.102763083Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-13T13:58:12.145822533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:58:12.14896083Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T13:58:14.159663839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T13:58:15.355570123Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-13T13:58:15.355591934Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-13T13:58:15.424662339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:58:15.427591538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:58:15.42760921Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:58:15.427629408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:58:15.429973614Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T13:58:16.269979978Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55206"
level=warn ts=2026-05-13T13:58:16.378863466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:58:18.609311464Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T13:58:20.659899807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:58:20.66441158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T13:58:21.378552083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:58:21.861528015Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-13T13:58:22.041586115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:58:22.044914227Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:58:22.044934561Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:58:22.044960685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:58:22.047553454Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T13:58:24.683431172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:58:24.687534498Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T13:58:25.115280183Z caller=checkpoint.go:372 msg="writing series" size="674 kB"
level=warn ts=2026-05-13T13:58:26.379193043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:28.017089778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:58:28.020127997Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T13:58:28.36732098Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-13T13:58:29.526230207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T13:58:31.378440299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:58:31.621248446Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=info ts=2026-05-13T13:58:31.634330059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:58:31.641277264Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:58:31.836368885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:58:31.839259606Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T13:58:34.874455016Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-13T13:58:34.874477886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-13T13:58:35.577193581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:58:35.58301046Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T13:58:36.379073536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:36.673509199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:58:36.679138801Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:58:36.727212859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:58:36.729571107Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T13:58:37.157229696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:58:37.162605187Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:58:37.692286734Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T13:58:38.127458791Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T13:58:38.224720206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:58:38.227230454Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T13:58:40.913053713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:58:40.915775238Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:58:41.378985382Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T13:58:41.379004414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:58:41.379013579Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:58:41.379024923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T13:58:41.38011602Z caller=checkpoint.go:384 msg="flushing series" totalSize="248 kB" series=1
level=info ts=2026-05-13T13:58:41.385086338Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-13T13:58:41.386232978Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006977619s
level=info ts=2026-05-13T13:58:41.833979388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:58:41.836650466Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T13:58:43.510429497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42624"
level=info ts=2026-05-13T13:58:43.885159025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T13:58:43.888017519Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T13:58:44.161084698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T13:58:46.271358337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58778"
level=warn ts=2026-05-13T13:58:46.378798271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:47.987897483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:58:47.991089476Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:58:47.991110287Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:58:47.991131299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:58:47.993543993Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:58:51.378417305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:51.683972306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:58:51.687523783Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:58:51.966209895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:58:51.969845484Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T13:58:51.969867795Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:58:51.969892273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:58:51.972811672Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:58:52.77720729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:58:52.781484263Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:58:54.277468731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:58:54.280677202Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T13:58:56.378752016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:58:57.501054547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:58:57.508311484Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T13:59:01.378522617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:59:01.459773781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:59:01.464980402Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T13:59:03.026220312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:59:03.029267119Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T13:59:06.234138636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:59:06.238890967Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T13:59:06.378749926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:59:06.513192347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:59:06.516633035Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T13:59:11.359636743Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T13:59:11.359825974Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T13:59:11.379195053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:59:11.379213168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T13:59:11.379213045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:59:11.379221016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T13:59:11.379228187Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T13:59:11.379340393Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-13T13:59:11.379348542Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T13:59:11.379364292Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.396µs success=true
level=debug ts=2026-05-13T13:59:11.379535195Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-13T13:59:13.09553641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T13:59:13.511957055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50892"
level=info ts=2026-05-13T13:59:14.104096529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:59:14.107237369Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:59:14.107254379Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:59:14.107273292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:59:14.109964567Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T13:59:14.162486083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T13:59:14.633582783Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-13T13:59:15.731390795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T13:59:15.740196443Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T13:59:15.848702633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:59:15.852134205Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T13:59:16.379407294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:17.886638153Z caller=checkpoint.go:372 msg="writing series" size="637 kB"
level=debug ts=2026-05-13T13:59:17.886658906Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-13T13:59:20.177218926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T13:59:20.182420131Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T13:59:21.139283538Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-13T13:59:21.378882101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:22.494619746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T13:59:24.386223615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T13:59:24.389188922Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T13:59:24.392422737Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-13T13:59:25.839287658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:59:25.84206187Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T13:59:26.379221789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:27.645591211Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-13T13:59:30.677943137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T13:59:30.684131166Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T13:59:30.74120532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T13:59:30.743978799Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T13:59:30.887368141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T13:59:30.890186064Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T13:59:30.898226671Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T13:59:31.378792005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:59:31.687136345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T13:59:31.692501219Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T13:59:34.151730019Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T13:59:35.180736663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T13:59:35.183848237Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T13:59:36.379211882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:37.404451654Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T13:59:38.040136825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T13:59:38.043809188Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T13:59:40.198759429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T13:59:40.202127284Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T13:59:40.657522393Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-13T13:59:40.725629682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T13:59:40.728720486Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T13:59:41.152962674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T13:59:41.156888761Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T13:59:41.378791345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T13:59:41.378816227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T13:59:41.37882686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T13:59:41.378857079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T13:59:42.279280206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T13:59:42.282526398Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T13:59:43.910113929Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-13T13:59:44.164102465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T13:59:46.145290209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T13:59:46.378770665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:47.163502665Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-05-13T13:59:50.416332809Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T13:59:50.416354668Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-13T13:59:51.378685166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:53.669127625Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T13:59:54.850689337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T13:59:54.859336177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T13:59:54.860095409Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T13:59:54.861918095Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T13:59:54.885985241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T13:59:54.888605943Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T13:59:56.332223341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T13:59:56.335111024Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T13:59:56.379275082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T13:59:56.92282641Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-13T13:59:58.363041306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:59:58.36671439Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:59:58.366735779Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T13:59:58.36675702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T13:59:58.369678768Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:00:00.175476748Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-13T14:00:00.46047227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:40948"
level=warn ts=2026-05-13T14:00:01.379158047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:00:01.620045705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:00:01.626794362Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:00:01.626813962Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:00:01.626840989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:00:01.633589339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:00:03.428817086Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=warn ts=2026-05-13T14:00:06.378680174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:06.68123516Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-13T14:00:09.935132341Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T14:00:09.935155493Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-13T14:00:11.359637242Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:00:11.359925262Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=debug ts=2026-05-13T14:00:11.361168123Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T14:00:11.361232237Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976311 n=1
level=debug ts=2026-05-13T14:00:11.361273582Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976311 n=1
level=info ts=2026-05-13T14:00:11.379270267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:00:11.379272302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:11.379287694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:00:11.379297216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:00:12.269697452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:00:12.27773926Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:00:13.187135847Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T14:00:13.51440443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:42268"
ts=2026-05-13T14:00:14.165889246Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:00:14.203722068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:00:14.207596537Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T14:00:16.276053992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:32962"
level=warn ts=2026-05-13T14:00:16.379291465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:16.440488605Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-13T14:00:17.944108388Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:00:17.985425884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:00:17.991526401Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:00:18.022944892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:00:18.024426403Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T14:00:18.137715216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:00:18.140445825Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:00:18.140467027Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:00:18.140488254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:00:18.140941239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:00:18.142691958Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:00:18.146298096Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:00:18.70493082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:00:18.707572818Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:00:19.693468383Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-13T14:00:21.379143956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:22.947010072Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=info ts=2026-05-13T14:00:23.454368358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:00:23.457633712Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:00:26.19915797Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-13T14:00:26.378685173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:29.452346542Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=debug ts=2026-05-13T14:00:29.452368688Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-13T14:00:29.907161082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:00:29.912628723Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:00:30.384218647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:00:30.387485372Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T14:00:31.379198553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:32.704925066Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T14:00:33.913726686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:00:33.917411343Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:00:34.475022098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:00:34.478967308Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T14:00:35.527053184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:00:35.530375341Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:00:35.540414047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:00:35.543002014Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:00:35.95846228Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T14:00:36.378705092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:00:36.940210731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:00:36.942873736Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:00:39.211290505Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=info ts=2026-05-13T14:00:39.929804352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:00:39.932658752Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T14:00:39.978968058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:00:39.981211754Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T14:00:41.37922403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:00:41.37922932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:00:41.379249891Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:00:41.379257804Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:00:42.116929639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:00:42.119779101Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:00:42.466872272Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T14:00:42.466893703Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=4
ts=2026-05-13T14:00:43.515799523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52830"
level=info ts=2026-05-13T14:00:43.706907093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:00:43.709818474Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
ts=2026-05-13T14:00:44.167488787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:00:45.718010734Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-13T14:00:45.751111126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:00:45.757491337Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:00:45.757508593Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:00:45.757531882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:00:45.763039763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T14:00:46.37940723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:00:47.441740431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:00:47.447855881Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:00:48.97072783Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T14:00:51.378368809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:00:51.44653668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:00:51.450030626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:00:52.224788932Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T14:00:52.22481324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-05-13T14:00:52.890805288Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:00:54.878058795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:00:54.883118801Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:00:55.476279447Z caller=checkpoint.go:372 msg="writing series" size="946 B"
level=warn ts=2026-05-13T14:00:56.378941539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:00:58.730617642Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T14:00:58.730637522Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T14:01:00.86757093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:01:00.870888217Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:01:01.378694334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:01.982195232Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T14:01:04.977030979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:01:04.985337491Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:01:05.235963719Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-13T14:01:05.280133424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:01:05.285701827Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T14:01:06.378715648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:01:08.201724687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:01:08.205038923Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:01:08.492248988Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T14:01:08.492266601Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=debug ts=2026-05-13T14:01:11.359314717Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:01:11.360365547Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:01:11.378743925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:01:11.378760402Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:01:11.378767783Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:01:11.378782013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:11.741380526Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-13T14:01:14.169240574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:01:14.995149669Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=info ts=2026-05-13T14:01:15.651461574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:01:15.654792388Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:01:15.654814566Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:01:15.65483427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:01:15.657522764Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T14:01:16.278839658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60196"
level=warn ts=2026-05-13T14:01:16.37913294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:17.012092924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:01:18.247740069Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T14:01:18.508569954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:01:18.511337623Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T14:01:21.379285144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:21.500628318Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T14:01:22.523939201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:01:22.527878995Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T14:01:22.793748596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:01:22.796836242Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:01:24.753467499Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=info ts=2026-05-13T14:01:24.85289095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:01:24.855477416Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:01:25.085351692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:01:25.088317605Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:01:25.768465556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:01:25.771279664Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T14:01:26.3785616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:01:26.950382864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:01:26.955490467Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:01:28.007893909Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T14:01:28.007915209Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
level=debug ts=2026-05-13T14:01:28.327389456Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:01:28.863904135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:01:28.870010435Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:01:29.96529221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:01:29.968258306Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:01:31.259718664Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-13T14:01:31.378969286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:01:32.206220772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:01:32.20926509Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:01:34.456605042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:01:34.45956133Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:01:34.512864277Z caller=checkpoint.go:372 msg="writing series" size="593 kB"
level=info ts=2026-05-13T14:01:35.118271865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:01:35.121262207Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T14:01:36.378625091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:37.765261333Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T14:01:37.853532945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:01:37.859970065Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:01:38.0824325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:01:38.08660433Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:01:38.752535994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:01:38.758152829Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:01:40.97596445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:01:40.979399115Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:01:41.018515392Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-13T14:01:41.378790643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:01:41.378793397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:41.37881405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:01:41.378823258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T14:01:44.170898568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:01:44.27181946Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
ts=2026-05-13T14:01:46.279633912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:44660"
level=warn ts=2026-05-13T14:01:46.3790037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:47.52509908Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=debug ts=2026-05-13T14:01:47.525123689Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=debug ts=2026-05-13T14:01:50.83121022Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T14:01:50.831234553Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=1
level=warn ts=2026-05-13T14:01:51.378418468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:54.030508858Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-13T14:01:56.378765935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:01:57.283923553Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-05-13T14:01:57.370200397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:01:57.996715158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:01:58.000919719Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:02:00.536868862Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-13T14:02:01.378497516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:02:03.128782839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:02:03.137103031Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:02:03.560691582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:02:03.565076091Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T14:02:03.789753037Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T14:02:05.86249272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:02:05.865769641Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:02:06.150639192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:02:06.153734809Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:02:06.248998106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:02:06.251422781Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T14:02:06.378597536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:07.042440547Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T14:02:08.69836489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:02:08.701107635Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:02:09.365164641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:02:09.370557788Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:02:09.370575683Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:02:09.370600648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:02:09.37524049Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:02:10.299083064Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-13T14:02:10.299105488Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=6
level=info ts=2026-05-13T14:02:11.359624726Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:02:11.359791012Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:02:11.379354153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:02:11.379358897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:11.379369445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:02:11.379377046Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:02:13.549155257Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
ts=2026-05-13T14:02:14.172160557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T14:02:16.37882423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:02:16.661971908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:02:16.66676186Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:02:16.666779183Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:02:16.666800576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:02:16.669463284Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:02:16.80158326Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T14:02:18.610314959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:02:18.617087925Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:02:19.15807876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:02:20.05466136Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=warn ts=2026-05-13T14:02:21.379243806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:02:21.530570564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:02:21.533616751Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:02:21.625937153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:02:21.630628067Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:02:22.614881916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:02:22.618116758Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:02:23.308312121Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T14:02:23.308334094Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-13T14:02:23.827613657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:02:25.918386113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:02:25.925297043Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T14:02:26.378626463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:26.561581697Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T14:02:26.561602853Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-13T14:02:28.284435598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:02:28.28762411Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:02:29.731782211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:02:29.73466418Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:02:29.825002292Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T14:02:29.825035141Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=warn ts=2026-05-13T14:02:31.379254909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:33.067043657Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-13T14:02:33.164369265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:02:33.167999947Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:02:34.479652267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:02:34.482596136Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:02:36.320541012Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T14:02:36.378759868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:02:36.511216775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:02:36.515269533Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:02:38.308585395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:02:38.3134681Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:02:39.574868548Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T14:02:39.574889281Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-13T14:02:41.379111421Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:02:41.379135299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:02:41.379144761Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:02:41.37913745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:42.826384575Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
ts=2026-05-13T14:02:44.173988471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:02:46.079062158Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-13T14:02:46.281867634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:43292"
level=warn ts=2026-05-13T14:02:46.379151173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:49.332656085Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T14:02:51.379133001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:02:51.65147679Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:02:52.584750705Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T14:02:54.229499013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:02:54.237005059Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:02:54.237035606Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:02:54.237061068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:02:54.242758352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:02:55.838045101Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-13T14:02:56.379140309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:02:57.981351789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:02:57.984203506Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:02:58.229924336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:02:58.233804065Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T14:02:59.09112964Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-13T14:03:00.331556621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:03:00.467937711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:54670"
level=info ts=2026-05-13T14:03:01.28160058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:03:01.287305704Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:03:01.378409526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:03:01.477728098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:03:01.480419622Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T14:03:02.344125611Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-13T14:03:04.053093411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:03:04.056038541Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:03:05.315154499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:03:05.318162654Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:03:05.59725946Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T14:03:05.92506392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:03:05.931251453Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:03:05.95751995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:03:05.960076325Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:03:06.055339068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:03:06.057920638Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T14:03:06.378632849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:03:07.672874043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:03:07.679711954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:03:08.850195413Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-13T14:03:09.05780326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:03:09.060797688Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:03:11.359662071Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:03:11.360079579Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:03:11.378334377Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:03:11.378356287Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:03:11.37836423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:03:11.378405074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:03:12.102942693Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
ts=2026-05-13T14:03:13.522396895Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34818"
ts=2026-05-13T14:03:14.17526558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:03:14.693630544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:03:14.700773722Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:03:15.355943362Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-13T14:03:15.355966102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-05-13T14:03:16.283111325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54494"
level=warn ts=2026-05-13T14:03:16.378526734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:03:16.79564978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:03:16.798333435Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:03:16.965689627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:03:16.968407513Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:03:17.203977456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:03:17.206580903Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:03:18.609530628Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-13T14:03:20.515145359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T14:03:21.378417507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:03:21.862661996Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-13T14:03:25.115590462Z caller=checkpoint.go:372 msg="writing series" size="687 kB"
level=warn ts=2026-05-13T14:03:26.379346715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:03:27.308752215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:03:27.314766576Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:03:27.891933233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:03:27.895168616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:03:28.367764449Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-13T14:03:31.37878295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:03:31.620945072Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=debug ts=2026-05-13T14:03:34.874429658Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-13T14:03:34.874452305Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-13T14:03:35.005388842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:03:35.012396816Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T14:03:36.37860034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:03:38.127090674Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-13T14:03:38.162377427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:03:38.168622955Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:03:38.168641465Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:03:38.16866334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:03:38.173398253Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:03:41.378998121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:03:41.379021941Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:03:41.379030671Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:03:41.379048807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:03:41.380094533Z caller=checkpoint.go:384 msg="flushing series" totalSize="248 kB" series=1
level=info ts=2026-05-13T14:03:41.387316016Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-13T14:03:41.388478987Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008990524s
ts=2026-05-13T14:03:44.1760566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:03:46.284667742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49382"
level=warn ts=2026-05-13T14:03:46.379238253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:03:47.615899449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:03:47.619648118Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:03:47.619665523Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:03:47.619685493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:03:47.622281718Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:03:49.594551784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:03:49.597904054Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:03:50.378117523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:03:50.381525532Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T14:03:51.3792551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:03:53.503914066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:03:54.72575238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:03:54.731262333Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:03:55.860609996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:03:55.865645568Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:03:55.906706208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:03:55.910469034Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T14:03:56.37911571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:03:57.442151964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:03:57.445343584Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T14:03:59.945122993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:03:59.947924095Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:04:00.245783402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:04:00.248865712Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T14:04:01.37866597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:01.505997048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:04:01.508859182Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:04:01.808918255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:04:01.964708698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:04:01.967365597Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:04:06.378796758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:07.538164539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:04:07.54529725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:04:08.429869039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:04:08.437420492Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:04:08.912643122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:04:08.915585527Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:04:09.951139317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:04:09.954046468Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:04:11.359677369Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:04:11.359882244Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:04:11.379283521Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T14:04:11.379299803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:11.379301875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:04:11.379324182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:04:11.379331638Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:04:11.379415093Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-13T14:04:11.379423123Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T14:04:11.379443817Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.418µs success=true
level=debug ts=2026-05-13T14:04:11.379625378Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T14:04:12.049166898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:04:12.0520128Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T14:04:13.525408227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47046"
ts=2026-05-13T14:04:14.176946434Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:04:14.632964533Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-13T14:04:14.696948697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:04:14.699635384Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T14:04:16.379195886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:04:17.886684488Z caller=checkpoint.go:372 msg="writing series" size="501 kB"
level=debug ts=2026-05-13T14:04:21.13978709Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-13T14:04:21.139809213Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T14:04:21.379039369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:24.103795161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:04:24.108031854Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:04:24.392158432Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-13T14:04:24.895364673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:04:24.898932237Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:04:25.749289235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:04:25.754402239Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-13T14:04:26.378795196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:04:27.644969808Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-13T14:04:29.18053053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:04:30.898146189Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-13T14:04:30.946292762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:04:30.953422246Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T14:04:31.379076403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:04:34.151228504Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-13T14:04:36.379372629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:04:37.404062086Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T14:04:40.320573287Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:04:40.321987732Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T14:04:40.657350435Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-13T14:04:41.123404177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:04:41.12624704Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:04:41.126266083Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:04:41.126290323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:04:41.128454675Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:04:41.378529093Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:04:41.378554971Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:04:41.378563024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:04:41.378625574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:41.645205504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:04:41.647699171Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-13T14:04:43.526555463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:41634"
level=debug ts=2026-05-13T14:04:43.910072279Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-13T14:04:44.177963538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T14:04:44.234664824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:04:44.239683201Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:04:44.390306649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:04:44.393167632Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T14:04:46.287835207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:42358"
level=warn ts=2026-05-13T14:04:46.37933625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:46.693450146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:04:46.696238173Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:04:46.696258387Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:04:46.696278819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:04:46.698423881Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:04:47.031758302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:04:47.034639466Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:04:47.163367666Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-13T14:04:50.416848067Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-13T14:04:50.856966917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:04:50.861156062Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T14:04:50.861196861Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:04:50.861218777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:04:50.864585382Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T14:04:51.379005876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:04:52.518861351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:04:52.522235158Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:04:53.668932743Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-13T14:04:53.668956384Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=10
level=info ts=2026-05-13T14:04:54.186158137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:04:54.189056828Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:04:54.289289268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:04:54.291959475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:04:54.813440096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:04:54.819445744Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:04:55.037992748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:04:55.044747736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:04:55.044861291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:04:55.047343956Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T14:04:56.379419696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:04:56.737012023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:04:56.922536214Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-05-13T14:04:57.552193517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:04:57.555836784Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:05:00.175007708Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-13T14:05:01.379379553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:03.428958584Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=info ts=2026-05-13T14:05:05.436393543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:05:05.441004806Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T14:05:06.379370662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:06.681636429Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T14:05:09.139700719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:05:09.144281621Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:05:09.144300696Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:05:09.14431939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:05:09.146770054Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:05:09.93450188Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T14:05:09.934522804Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-13T14:05:11.359381903Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:05:11.360453034Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:05:11.378821101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:05:11.378834767Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:05:11.378842122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:05:11.378833214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:13.187537777Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T14:05:13.72193971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:05:13.724965858Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:05:13.724986821Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:05:13.725008301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:05:13.727413892Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T14:05:14.178942879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:05:15.042540146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:05:15.046068834Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:05:15.826367883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:05:15.829203875Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T14:05:16.289145351Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33006"
level=warn ts=2026-05-13T14:05:16.379206442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:16.440332782Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T14:05:17.455996904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:05:17.458716876Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:05:18.413659017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:05:18.418497032Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:05:18.86473635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:05:18.869737793Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:05:19.059505458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:05:19.69344511Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-13T14:05:21.37919013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:05:22.455278756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:05:22.458041611Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:05:22.946548457Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=info ts=2026-05-13T14:05:24.14469001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:05:24.147592928Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:05:25.770216831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:05:25.773008837Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T14:05:25.773029232Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:05:25.773050434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:05:25.7752239Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T14:05:26.063017033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:05:26.073223384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:05:26.076771578Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:05:26.199198342Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-13T14:05:26.373748939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="889 kB" avg_uncomp="889 kB" idle=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:05:26.376960188Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=warn ts=2026-05-13T14:05:26.379022834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:05:26.404097695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:05:26.408766522Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:05:29.452495648Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=debug ts=2026-05-13T14:05:29.452519376Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-13T14:05:30.474025037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:43182"
level=warn ts=2026-05-13T14:05:31.3791042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:05:32.011141387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:05:32.014386573Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:05:32.705400559Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T14:05:33.655088796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:05:33.66315462Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:05:33.663188126Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:05:33.663208615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:05:33.665681989Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:05:34.254355055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:05:34.257471156Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:05:35.957958447Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T14:05:36.379191421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:05:37.730727082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:05:37.733792607Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T14:05:37.899232912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:05:37.902406788Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:05:39.211997862Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=info ts=2026-05-13T14:05:40.170743199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:05:40.177069162Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T14:05:40.603224529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:05:40.606611594Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:05:41.379365633Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:05:41.379374982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:41.379390035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:05:41.379403535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:05:42.466230692Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T14:05:42.466252202Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=4
level=info ts=2026-05-13T14:05:42.474868688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:05:42.47855346Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
ts=2026-05-13T14:05:44.18050002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:05:44.185205716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:05:44.188112756Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:05:44.504129896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:05:44.506991226Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:05:45.717222057Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-13T14:05:46.378561692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:05:46.634229714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:05:46.640434129Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:05:47.142618245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:05:47.148688236Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:05:47.838690416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:05:47.841929245Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:05:48.970243966Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-13T14:05:51.152792739Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T14:05:51.379236728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:52.224105053Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T14:05:52.224125095Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-13T14:05:54.181066238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:05:54.184286436Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:05:54.184302807Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:05:54.184322167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:05:54.186532771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:05:55.477025585Z caller=checkpoint.go:372 msg="writing series" size="946 B"
level=info ts=2026-05-13T14:05:55.911127341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:05:55.914020017Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T14:05:56.378942835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:05:58.730599546Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T14:05:58.730622003Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T14:05:59.600867581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:05:59.603886308Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:06:00.255761734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:06:00.259469778Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T14:06:00.475418713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:32842"
level=debug ts=2026-05-13T14:06:01.021210093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T14:06:01.378448021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:01.982938045Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T14:06:04.669857527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:06:04.672823862Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T14:06:05.235350156Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-13T14:06:06.378685831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:06:07.1847031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:06:07.187740194Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:06:08.190414999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:06:08.195556372Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:06:08.492855971Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T14:06:08.49288339Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=info ts=2026-05-13T14:06:08.588761618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:06:08.591899851Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:06:09.541962178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:06:09.545986684Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:06:10.297926361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:06:10.302740828Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:06:11.359636832Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:06:11.359647139Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:06:11.379077972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:06:11.379093599Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:06:11.379101379Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:06:11.379146238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:11.741298059Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-13T14:06:12.050103157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:06:12.05324155Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
ts=2026-05-13T14:06:14.181718612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:06:14.994921726Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=warn ts=2026-05-13T14:06:16.379230966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:17.250012057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:06:18.247625183Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T14:06:18.712732033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:06:18.718022855Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:06:20.905966502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:06:20.908819886Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T14:06:21.379362178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:21.500685347Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T14:06:22.830860351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:06:22.835932989Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:06:24.753795541Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=info ts=2026-05-13T14:06:24.774758873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:06:24.778208239Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:06:26.255142231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:06:26.258108562Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:06:26.316167234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:06:26.318840769Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T14:06:26.379054634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:28.009005585Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T14:06:28.009026731Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
level=info ts=2026-05-13T14:06:28.269948614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:06:28.276558546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:06:31.260277984Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-13T14:06:31.378878741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:06:32.503057324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:06:32.507083423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T14:06:34.513076958Z caller=checkpoint.go:372 msg="writing series" size="587 kB"
level=warn ts=2026-05-13T14:06:36.378945337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:37.765514266Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T14:06:41.019029839Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-13T14:06:41.379355452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:06:41.379364828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:06:41.379384646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:06:41.379392793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:06:41.759382141Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:06:42.697324891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:06:42.705134686Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:06:42.705156464Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:06:42.705197903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:06:42.71052139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:06:43.444045079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:06:43.44642677Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:06:43.446443712Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:06:43.446463213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:06:43.448422688Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T14:06:44.183437882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:06:44.271273453Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-13T14:06:44.593217242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:06:44.595830017Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:06:44.787803492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:06:45.318050174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:06:45.321305208Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:06:45.321324609Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:06:45.321346062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:06:45.323976412Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:06:46.37895109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:06:47.524789975Z caller=checkpoint.go:372 msg="writing series" size="443 kB"
level=debug ts=2026-05-13T14:06:47.524811878Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-13T14:06:49.764932436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:06:49.768154477Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:06:50.385887823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:06:50.388799614Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:06:50.835736401Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T14:06:50.835762001Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=1
level=warn ts=2026-05-13T14:06:51.378973012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:06:53.82973482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:06:53.833435376Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:06:54.031019532Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-13T14:06:56.13613376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:06:56.140352073Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:06:56.378499092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:06:56.942098336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:06:56.945445426Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:06:57.102961523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:06:57.111003385Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:06:57.283776474Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=info ts=2026-05-13T14:06:58.304459799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:06:58.307947411Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:06:58.383362055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:06:58.386436034Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:07:00.536755324Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=info ts=2026-05-13T14:07:00.97419099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:07:00.976945079Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:07:01.37905704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:07:01.794966178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:07:01.797918452Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T14:07:02.114065684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:07:02.116795382Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:07:02.59198198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:07:02.59771075Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:07:03.790320585Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T14:07:06.060076419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:07:06.065344452Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-13T14:07:06.379228752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:07:07.026058228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:07:07.029089365Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:07:07.04321806Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T14:07:08.206614419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:07:10.298702611Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-13T14:07:10.298724257Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=6
level=debug ts=2026-05-13T14:07:11.359459386Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:07:11.360504641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:07:11.378873475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:07:11.378897042Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:07:11.378915856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:07:11.378920088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T14:07:13.532627463Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57716"
level=debug ts=2026-05-13T14:07:13.549255413Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-13T14:07:14.184299766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=warn ts=2026-05-13T14:07:16.378969372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:07:16.802026906Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T14:07:17.216223419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:07:17.220012195Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:07:17.942953135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:07:17.947003433Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:07:20.05514505Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-13T14:07:20.659873198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:07:20.665365791Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:07:21.083434168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:07:21.086702427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T14:07:21.378883818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:07:23.308057434Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T14:07:23.308091754Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T14:07:23.373133615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:07:23.377461409Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T14:07:26.378743928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:07:26.561218066Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T14:07:26.561236338Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-13T14:07:27.490058798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:07:27.497456058Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:07:27.497475222Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:07:27.497499773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:07:27.503585726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:07:29.826754185Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T14:07:29.826779726Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-13T14:07:30.302453922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:07:30.309091378Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T14:07:30.479061662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:46828"
level=warn ts=2026-05-13T14:07:31.378841654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:07:32.239161366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:07:32.242058249Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:07:33.067056389Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-13T14:07:35.586373135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:07:35.589641632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:07:36.319767875Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T14:07:36.378698815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:07:38.663392082Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:07:39.574557272Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T14:07:39.574578365Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=debug ts=2026-05-13T14:07:40.388664396Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T14:07:41.37915295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:07:41.37916495Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:07:41.379198397Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:07:41.379206367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:07:42.098135045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:07:42.101037318Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:07:42.411063549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:07:42.41792022Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:07:42.617514109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:07:42.620296285Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:07:42.826211836Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=info ts=2026-05-13T14:07:44.003256526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:07:44.01691348Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T14:07:44.185820018Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:07:44.416216749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:07:44.421992797Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:07:45.177958144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:07:45.180737555Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:07:46.079334232Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T14:07:46.379226734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:07:46.498535608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:07:46.50376849Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:07:48.45251416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:07:48.455466102Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:07:49.331981452Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-13T14:07:50.009745661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:07:50.013597774Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:07:51.0305533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:07:51.033461698Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T14:07:51.378709908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:07:52.585323717Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T14:07:54.381579911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:07:54.384676955Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:07:54.384696518Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:07:54.384715514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:07:54.387277932Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:07:55.838145202Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-13T14:07:55.992576627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:07:55.998342343Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-13T14:07:56.379052714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:07:59.091454361Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-13T14:08:01.378995646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:02.344372612Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=info ts=2026-05-13T14:08:03.139510111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:08:03.147119035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:08:04.929066529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:08:04.934594244Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:08:05.596673778Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-13T14:08:06.378764988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:06.454224065Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:08:08.850388569Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-13T14:08:10.094619423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:08:10.098052832Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:08:10.219446889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:08:10.223467964Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:08:11.359639322Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:08:11.359832863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=warn ts=2026-05-13T14:08:11.379213305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:08:11.379218096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:08:11.379232065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:08:11.379239284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:08:11.880663894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:08:11.883259293Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:08:12.103308762Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-13T14:08:13.030993543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:08:13.037380168Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T14:08:13.535459104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:48050"
ts=2026-05-13T14:08:14.187276641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:08:15.355997398Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-13T14:08:15.356018146Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=info ts=2026-05-13T14:08:15.687943097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:08:15.690732519Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:08:16.379378119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:18.609563765Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-13T14:08:20.956711006Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:08:20.958404792Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T14:08:21.378905787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:21.862534935Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T14:08:23.343429454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:08:23.347603125Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T14:08:25.116233268Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
level=info ts=2026-05-13T14:08:26.166020558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:08:26.169158248Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T14:08:26.378804513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:28.36802093Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-13T14:08:28.476320065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:08:28.479535279Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T14:08:30.481997793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:33046"
level=info ts=2026-05-13T14:08:31.269682607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:08:31.272617137Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T14:08:31.378810888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:31.620603699Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=info ts=2026-05-13T14:08:32.514639884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:08:32.517527493Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:08:33.418210096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:08:33.421053248Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:08:34.874897948Z caller=checkpoint.go:372 msg="writing series" size="640 kB"
level=debug ts=2026-05-13T14:08:34.874920221Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-13T14:08:35.775986518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:08:35.779129775Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:08:36.159152424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:08:36.16237607Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=warn ts=2026-05-13T14:08:36.378900989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:38.127616172Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-13T14:08:39.693654099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:08:40.478867133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:08:40.483064081Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:08:40.619471021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:08:40.625502426Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:08:41.379290165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:08:41.379313293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:08:41.379321366Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:08:41.379334997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:08:41.380370514Z caller=checkpoint.go:384 msg="flushing series" totalSize="202 kB" series=1
level=info ts=2026-05-13T14:08:41.385005506Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-13T14:08:41.386207347Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006627998s
ts=2026-05-13T14:08:43.536896491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:60062"
level=info ts=2026-05-13T14:08:43.861317691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:08:43.864087637Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T14:08:44.188083559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T14:08:45.728980639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:08:45.73387613Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:08:45.733894846Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:08:45.733916305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:08:45.738558119Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:08:46.378699517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:08:47.415651616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:08:47.420187801Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:08:49.03460163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:08:49.040021808Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:08:50.948224321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:08:50.95122101Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T14:08:51.378471459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:08:54.059906856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:08:54.067422887Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:08:56.29975885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:08:56.307771471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T14:08:56.379133597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T14:09:00.483603111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:38968"
level=warn ts=2026-05-13T14:09:01.378403286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:02.390909383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:09:02.395615249Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:09:02.451802557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:09:02.454939755Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:09:03.425412941Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:09:04.305696515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:09:04.308776526Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:09:05.835622356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:09:05.835669382Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T14:09:05.839001373Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T14:09:06.379206465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:09.84145218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:09:09.844158615Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:09:09.893367554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:09:09.89681563Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T14:09:11.359253692Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:09:11.360284563Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:09:11.378645621Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:09:11.378663018Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:09:11.378672754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:09:11.378676978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:11.378677976Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T14:09:11.378817369Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-13T14:09:11.378826303Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T14:09:11.378849513Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.096µs success=true
level=debug ts=2026-05-13T14:09:11.379044166Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-13T14:09:13.538011009Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:56954"
ts=2026-05-13T14:09:14.189696943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T14:09:14.257531185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:09:14.261165474Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:09:14.261194905Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:09:14.261214472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:09:14.263488451Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:09:14.632856821Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=info ts=2026-05-13T14:09:16.309529174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:09:16.312584279Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:09:16.37838324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:17.132030099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:09:17.135480637Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:09:17.885932027Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-13T14:09:17.885955084Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-13T14:09:21.138669344Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-13T14:09:21.378965668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:21.802470029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:09:21.806672975Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:09:22.792595281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:09:22.798695416Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:09:22.798714918Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:09:22.798740186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:09:22.804086589Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:09:23.307356463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:09:23.310317078Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:09:23.614283402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:09:23.617446094Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:09:24.392122348Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-13T14:09:25.812294157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:09:25.815270452Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:09:25.929455894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:09:25.93185415Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:09:26.379063884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:26.859027548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:09:26.86248212Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:09:27.644406473Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-13T14:09:29.543655986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:09:29.546845105Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:09:29.576087444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:09:29.578622564Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-13T14:09:30.484575854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:56996"
level=debug ts=2026-05-13T14:09:30.897144689Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T14:09:31.379138207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:32.083548057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:09:32.089379531Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:09:33.573895742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:09:33.576673583Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T14:09:34.151151868Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T14:09:34.904202581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:09:34.907042121Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:09:35.056494009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:09:35.061349579Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-13T14:09:36.378887027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:09:37.094620419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:09:37.402648257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:09:37.403731499Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=error ts=2026-05-13T14:09:37.408844248Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:09:37.87010204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:09:37.875325177Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:09:39.236032062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:09:39.238991879Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:09:40.656308756Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-13T14:09:41.379003569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:09:41.379025859Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:09:41.379033739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:09:41.379043235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:41.945126954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:09:41.951087655Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T14:09:43.539574229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:57116"
level=debug ts=2026-05-13T14:09:43.909688599Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-13T14:09:44.190721389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
ts=2026-05-13T14:09:46.300858114Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:47278"
level=warn ts=2026-05-13T14:09:46.379034495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:09:47.162898629Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-13T14:09:49.585313049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:09:49.589383089Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:09:50.415807558Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T14:09:50.415828393Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-13T14:09:51.379124403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:52.830467651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:09:52.833461601Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:09:53.668920718Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T14:09:54.220440449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:09:54.223502986Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:09:56.37935711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:09:56.513646277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:09:56.517651646Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T14:09:56.921628964Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-05-13T14:09:57.173505699Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:10:00.174775204Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-13T14:10:01.379219894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:03.427607287Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=info ts=2026-05-13T14:10:04.328003101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:10:04.33234171Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:10:04.841815019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:10:04.845509587Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:10:05.809225027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:10:05.812564153Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:10:06.378995069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:06.680765123Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-13T14:10:08.502192818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:10:08.502256794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:10:08.506891788Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:10:08.630270638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:10:08.633758563Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:10:09.131989738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:10:09.134967022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:10:09.934225707Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T14:10:09.934248895Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-13T14:10:11.359705688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:10:11.359699585Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:10:11.379134982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:10:11.379151271Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T14:10:11.379144332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:10:11.379158754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:10:12.888956431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:10:12.892047806Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:10:13.186929461Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T14:10:13.540444184Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:52366"
ts=2026-05-13T14:10:14.1921487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:10:15.76930642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:10:15.772283254Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T14:10:16.302353811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:53462"
level=warn ts=2026-05-13T14:10:16.378966141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:16.440131348Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T14:10:16.981069169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:10:16.983999369Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:10:16.984023291Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:10:16.984052242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:10:16.986607856Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:10:17.838129577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:10:17.843640016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:10:18.964964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:10:18.969591859Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:10:19.692652672Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-13T14:10:20.217056281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:10:20.219907564Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T14:10:21.378718013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:22.9468221Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=info ts=2026-05-13T14:10:22.95767493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:10:22.961462564Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:10:24.267119816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:10:24.2700346Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T14:10:25.80541806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:10:25.8086399Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:10:26.1987446Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-13T14:10:26.379341036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:29.452980607Z caller=checkpoint.go:372 msg="writing series" size="497 kB"
level=debug ts=2026-05-13T14:10:29.453002407Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T14:10:31.379265217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:10:32.343729644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.4 MB" total_uncomp="108 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:10:32.349964797Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:10:32.705060454Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T14:10:33.190485008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:10:33.193592577Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:10:35.013978762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:10:35.01929829Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:10:35.019326961Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:10:35.019350342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:10:35.023724798Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:10:35.032752471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:10:35.034989377Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:10:35.957787657Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T14:10:36.378768825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:10:36.722520261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:10:36.725797393Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:10:39.211651442Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=info ts=2026-05-13T14:10:40.794755715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:10:40.80100775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:10:40.982559105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:10:40.986560321Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T14:10:41.378350275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:10:41.378372341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:10:41.378380245Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:10:41.378386328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:42.46573684Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-13T14:10:42.465758453Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=debug ts=2026-05-13T14:10:43.293002242Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:10:43.541692306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:45984"
level=debug ts=2026-05-13T14:10:44.033742422Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:10:44.033741354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="352 kB" avg_comp="352 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:10:44.043796169Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
ts=2026-05-13T14:10:44.193508615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:10:45.717112572Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-13T14:10:46.140380908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:10:46.143505395Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T14:10:46.303287661Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33042"
level=warn ts=2026-05-13T14:10:46.378865835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:10:47.442381276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:10:47.445216969Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:10:48.133955086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:10:48.137254669Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:10:48.709107402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:10:48.712819995Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:10:48.845213109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:10:48.848879691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:10:48.970333573Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T14:10:50.054435996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:10:50.057278948Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T14:10:51.379012232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:10:52.223961372Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T14:10:52.223981832Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-13T14:10:53.153112493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:10:53.162264745Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:10:54.323401293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:10:54.326751886Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:10:55.476119451Z caller=checkpoint.go:372 msg="writing series" size="946 B"
level=info ts=2026-05-13T14:10:55.634561019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:10:55.637351493Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:10:55.637374771Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:10:55.637396075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:10:55.639501455Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T14:10:56.379057203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:10:58.115070452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:10:58.117996024Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:10:58.7290955Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T14:10:58.729115824Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T14:11:00.477530937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:11:00.48102988Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
ts=2026-05-13T14:11:00.48777526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:58082"
level=info ts=2026-05-13T14:11:00.691904468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:11:00.694508503Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T14:11:01.378610056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:11:01.98279218Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T14:11:03.411575274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:11:03.414273395Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:11:03.414290503Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:03.414315488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:11:03.417060008Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:11:05.067643525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:11:05.072169784Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T14:11:05.235522833Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-13T14:11:06.378560648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:11:08.490800767Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T14:11:08.490822856Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=info ts=2026-05-13T14:11:11.359644802Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:11:11.359654565Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:11:11.379074801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:11:11.379079076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:11:11.379093225Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:11:11.379102656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:11:11.741068422Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-13T14:11:13.023513664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:11:13.026682812Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:11:13.832770296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:11:13.835780501Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T14:11:14.19522026Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:11:14.99430969Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-05-13T14:11:16.059098866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:11:16.062574124Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T14:11:16.379351898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:11:17.49369623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:11:17.50021887Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:11:18.034788883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:11:18.039751638Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:11:18.247467628Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T14:11:18.526805787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:11:18.529481162Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:11:18.948815953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:11:18.951860106Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:11:19.473620405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:11:19.923052054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:11:20.380355762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:11:20.38334347Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T14:11:21.378846816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:11:21.500136888Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T14:11:24.753017929Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=info ts=2026-05-13T14:11:25.098796293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="779 kB" avg_comp="779 kB" total_uncomp="6.2 MB" avg_uncomp="6.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T14:11:25.104891791Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e215e24c8%3A19e218af48c%3Ac39992f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T14:11:25.104916774Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:25.104945829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:11:25.108504377Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T14:11:26.378751202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:11:26.89138294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:11:26.895065954Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:11:28.007868763Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T14:11:28.007891664Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=info ts=2026-05-13T14:11:28.318810068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:11:28.323623966Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:11:31.207883197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:11:31.214471115Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:11:31.258927703Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T14:11:31.378634122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:11:32.057053184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:11:32.062997019Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:11:33.31548155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:11:33.318348445Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:11:33.318367004Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:33.318387684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:11:33.320706954Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:11:34.512471872Z caller=checkpoint.go:372 msg="writing series" size="604 kB"
level=warn ts=2026-05-13T14:11:36.379373275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:11:37.609752275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:11:37.612644951Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:11:37.765301486Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T14:11:38.580560897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:11:38.584431146Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:11:38.58445331Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:38.584476556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:11:38.587390597Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:11:39.33645892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:11:39.339740361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:11:40.891336933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:11:40.894444796Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:11:41.017849231Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-13T14:11:41.379132978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:11:41.379161491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:11:41.379197812Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:11:41.379213258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T14:11:43.543848265Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:51086"
level=info ts=2026-05-13T14:11:43.848367384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:11:43.851219812Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:11:44.187019892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:11:44.190795301Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T14:11:44.19595107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:11:44.271566009Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-13T14:11:44.271580509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T14:11:44.49622467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:11:44.499061607Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:11:45.066596125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:11:45.06973143Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:11:45.06974819Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:45.069768341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:11:45.072295017Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
ts=2026-05-13T14:11:46.30674068Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:54264"
level=warn ts=2026-05-13T14:11:46.379159573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:11:47.523869604Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=info ts=2026-05-13T14:11:48.145324798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:11:48.148456054Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:11:50.835778261Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T14:11:50.835802069Z caller=checkpoint.go:384 msg="flushing series" totalSize="55 MB" series=2
level=warn ts=2026-05-13T14:11:51.379036281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:11:51.5591043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:11:52.465543288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:11:52.469392612Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:11:54.029790723Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-13T14:11:56.30482327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:11:56.310379026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:11:56.310400361Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:56.310419935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:11:56.313240781Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T14:11:56.378845674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:11:56.509126968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T14:11:56.512158153Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e215eb9e5%3A19e215ef807%3Af26f1879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T14:11:56.512195537Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:11:56.611109764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:11:56.614583061Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:11:57.12249052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:11:57.126337605Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:11:57.283364909Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=info ts=2026-05-13T14:11:58.423205468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:11:58.429086132Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:11:59.246297175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:11:59.249532247Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:12:00.08373132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:12:00.087358326Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:12:00.536900898Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-13T14:12:01.378929783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:03.789289423Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T14:12:04.241324254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:12:04.248212088Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:12:06.378793667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:07.042554374Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T14:12:07.233195899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:12:07.238227679Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:12:08.138160621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:12:08.142329686Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T14:12:08.547620271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:12:08.550642043Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:12:10.042542181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:12:10.045570084Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:12:10.298498078Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T14:12:10.298517239Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=6
level=info ts=2026-05-13T14:12:10.882131835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:12:10.885540874Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:12:11.359636812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:12:11.359911537Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:12:11.379291255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:12:11.379307117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:12:11.37931506Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:12:11.379303636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:12:12.069893626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:12:12.073559426Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=info ts=2026-05-13T14:12:12.410569202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:12:12.417610384Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:12:12.764741243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:12:12.767545437Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:12:12.964224192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:12:12.966918078Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:12:12.966933989Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:12:12.966953395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:12:12.969025011Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T14:12:13.54559297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:40512"
level=debug ts=2026-05-13T14:12:13.548503533Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
ts=2026-05-13T14:12:14.197401988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=warn ts=2026-05-13T14:12:16.378884352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:16.80373296Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T14:12:17.192906819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:12:17.196651446Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:12:18.019546464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:12:18.022196331Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:12:19.930468107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:12:19.932274252Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T14:12:20.054299846Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=warn ts=2026-05-13T14:12:21.37861896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:23.127846304Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:12:23.28039687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:12:23.283239752Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:12:23.283258327Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:12:23.283277616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:12:23.28561011Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:12:23.298677242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:12:23.301305504Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:12:23.307935179Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T14:12:23.307948318Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T14:12:23.362749764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:12:23.368834046Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:12:24.904972749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:12:24.908770498Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:12:25.742903479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:12:25.746642193Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T14:12:26.379041162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:26.560763906Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T14:12:26.560783891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-13T14:12:29.777678048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:12:29.784506163Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:12:29.826651657Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-13T14:12:29.826671838Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-05-13T14:12:30.91985318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:12:30.923081246Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:12:31.06055593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:12:31.063320489Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T14:12:31.379148238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:33.067019687Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-13T14:12:33.152136015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:12:33.15587621Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T14:12:36.319929405Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T14:12:36.378882042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:12:36.755503079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:12:36.758144736Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:12:39.575874439Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T14:12:39.575895435Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-13T14:12:40.133041831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:12:40.136315166Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:12:40.18165535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:12:40.184399372Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:12:41.378907719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:12:41.378904223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:12:41.378932725Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:12:41.378941587Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:12:42.826144856Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=info ts=2026-05-13T14:12:42.892037871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:12:42.895291946Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T14:12:44.198702185Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:12:46.07811571Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-05-13T14:12:46.276570992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:12:46.282708133Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:12:46.37898113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:12:47.195662492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:12:47.19874809Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:12:48.710088641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:12:48.712985168Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:12:49.331918108Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T14:12:50.159410856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T14:12:51.37927161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:12:52.583949692Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T14:12:53.242560315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:12:53.246533096Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=info ts=2026-05-13T14:12:55.235303432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:12:55.239700475Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T14:12:55.34205896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:12:55.345073873Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:12:55.837593821Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-13T14:12:56.0753731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:12:56.081258262Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T14:12:56.37918807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:12:59.060194677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:12:59.064088618Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:12:59.090441404Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-13T14:12:59.592021978Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:13:00.116446615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:13:00.119088426Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:13:01.378756422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:01.665099225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:13:01.672012923Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:13:02.343576857Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-13T14:13:03.705443919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:13:03.708662983Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:13:04.094788388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:13:04.098036936Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:13:05.596008498Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-13T14:13:05.596030203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T14:13:06.379044456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:08.425352711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:13:08.431278686Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:13:08.43129576Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:13:08.43131694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:13:08.434394965Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:13:08.849707775Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-13T14:13:10.279686082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:13:10.285760074Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:13:11.359648594Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:13:11.360037556Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:13:11.378327317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:13:11.378365125Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T14:13:11.378374978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:11.378382982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:13:11.922160192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:13:11.929758885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:13:11.929778836Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:13:11.929801895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:13:11.93641429Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:13:11.979764482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:13:11.982611501Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:13:12.102948691Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
ts=2026-05-13T14:13:14.200109312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:13:15.355329834Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
ts=2026-05-13T14:13:16.310814406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:49284"
level=warn ts=2026-05-13T14:13:16.379137531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:16.572208513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:13:16.574703026Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:13:18.269082719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:13:18.272267499Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:13:18.487938516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:13:18.490926224Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:13:18.608292911Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T14:13:19.011458907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:13:19.015115231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-13T14:13:21.37842573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:21.843389675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:13:21.846318825Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:13:21.861424848Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-13T14:13:22.881935921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:13:22.890070779Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:13:25.114762124Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=debug ts=2026-05-13T14:13:25.629618055Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:13:25.629654772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="43 kB" avg_uncomp="43 kB" idle=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:13:25.632944664Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:13:26.378301064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:13:26.378387198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-13T14:13:26.381831149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:13:28.118448372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:13:28.121405316Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:13:28.121426317Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:13:28.121449382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:13:28.123746268Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:13:28.367800082Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-13T14:13:29.577151652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:13:29.58208451Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=warn ts=2026-05-13T14:13:31.379146685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:13:31.620310297Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=debug ts=2026-05-13T14:13:34.874226667Z caller=checkpoint.go:372 msg="writing series" size="728 kB"
level=debug ts=2026-05-13T14:13:34.874241878Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=warn ts=2026-05-13T14:13:36.378759267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:36.390890519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:13:36.393798835Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:13:37.525526121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:13:37.528472905Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:13:38.127145063Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-13T14:13:38.44722191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:13:38.450785142Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:13:39.156165356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:13:39.159561831Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:13:39.260788659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:13:39.264101926Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:13:39.329836622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:13:40.277689019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:13:40.280378693Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:13:41.3784115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:13:41.378436173Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:13:41.37844516Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:13:41.378448164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:13:41.379504105Z caller=checkpoint.go:384 msg="flushing series" totalSize="202 kB" series=1
level=info ts=2026-05-13T14:13:41.383789071Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-13T14:13:41.384978134Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005983984s
ts=2026-05-13T14:13:44.201993746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:13:45.098130104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:13:45.101247568Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:13:45.489380319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:13:45.492465752Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:13:45.502529423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:13:45.507957399Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:13:46.379292277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:46.990367809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:13:46.994971294Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T14:13:48.987955348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:13:48.991042678Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:13:49.251770343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:13:49.257575567Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:13:50.971085277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:13:50.974556273Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T14:13:51.378480387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:52.667105783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:13:52.674418348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:13:52.674443631Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:13:52.674471798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:13:52.682045125Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T14:13:56.379245743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:13:56.623936884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:13:56.626864021Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:13:58.490042978Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:13:58.799912235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:13:58.80667425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:14:00.184219229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:14:00.187581826Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:14:00.353971432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:14:00.358573359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:14:00.370760534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:14:00.373417551Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-13T14:14:01.378926322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:06.118217586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:14:06.122274652Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:14:06.378610544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:08.142048636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:14:08.148653258Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:14:08.390412241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:14:08.39341548Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:14:08.657092757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:14:08.6615446Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:14:09.871876041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:14:09.874778628Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:14:11.359791484Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:14:11.359812723Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:14:11.379192598Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:14:11.379208624Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:14:11.379216896Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:14:11.379223564Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T14:14:11.37926729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:11.379356342Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-13T14:14:11.379363189Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T14:14:11.379382633Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.808µs success=true
level=debug ts=2026-05-13T14:14:11.379565483Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-13T14:14:14.203434007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=debug ts=2026-05-13T14:14:14.633399231Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-13T14:14:14.816856265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:14:14.820245064Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:14:16.232216152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:14:16.234801636Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:14:16.378929115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:14:17.886346917Z caller=checkpoint.go:372 msg="writing series" size="591 kB"
level=debug ts=2026-05-13T14:14:21.13890418Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-13T14:14:21.379015112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:22.671932486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:14:22.675403913Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:14:22.716444333Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:14:24.392424109Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-13T14:14:24.392445913Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-05-13T14:14:26.379458178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:14:26.63402149Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:14:27.07446439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:14:27.078134955Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:14:27.64479431Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-13T14:14:28.501015778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:14:28.503754819Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:14:30.898479458Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-13T14:14:31.073103657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:14:31.077325202Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T14:14:31.379202666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:34.001105482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:14:34.003959006Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:14:34.151375156Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T14:14:35.822887056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:14:35.826161382Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T14:14:36.37919223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:14:37.404221731Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-13T14:14:38.73894522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:14:38.742568494Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:14:39.048642254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:14:39.055418384Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:14:40.559581485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:14:40.564347636Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:14:40.657609883Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-05-13T14:14:41.378371027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:41.378377914Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:14:41.378399378Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:14:41.378407249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:14:41.721383223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:14:41.7275692Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:14:43.910030704Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-05-13T14:14:43.980216874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:14:43.983590432Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T14:14:44.204387235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:14:44.209070755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:14:44.211881562Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:14:44.211898113Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:14:44.211916478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:14:44.214275309Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:14:44.500082154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:14:44.505298272Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:14:45.678463159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:14:45.681228743Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:14:46.378740954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:46.924277548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:14:46.927966604Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T14:14:47.163912071Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-13T14:14:47.984993914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:14:47.989997586Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:14:49.119153655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:14:49.122880367Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:14:50.002501069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:14:50.005461626Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:14:50.416875945Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T14:14:51.379008011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:14:51.808989637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:14:51.811827885Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:14:53.669084556Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T14:14:56.379026302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:14:56.922362631Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-13T14:14:59.12150451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:15:00.175505275Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-13T14:15:00.457373501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:15:00.460250111Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:15:00.680831841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:15:00.684273454Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:15:01.378814601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:01.482114336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:15:01.485512553Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:15:02.496617611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:15:02.499901431Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:15:03.131417711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:15:03.134155655Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:15:03.428393645Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-13T14:15:03.42841461Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-05-13T14:15:04.164451676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:15:04.167515599Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:15:06.378377109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:15:06.681429085Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T14:15:07.79372379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:15:07.797354005Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:15:09.934553313Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=info ts=2026-05-13T14:15:11.359626839Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:15:11.35983708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=debug ts=2026-05-13T14:15:11.361093263Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T14:15:11.361137946Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976312 n=1
level=debug ts=2026-05-13T14:15:11.361194055Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976312 n=1
level=info ts=2026-05-13T14:15:11.37923311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:15:11.379251199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:15:11.379260245Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:15:11.3792586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:15:13.187445456Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T14:15:13.55391137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:47796"
level=debug ts=2026-05-13T14:15:13.653249961Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T14:15:14.205941566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:15:15.861126734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:15:15.864137754Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T14:15:16.316190767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:38384"
level=warn ts=2026-05-13T14:15:16.379162595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:15:16.440325351Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-13T14:15:16.440344127Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T14:15:17.661130416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:15:17.664305706Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:15:19.693781717Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-13T14:15:21.379195917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:22.803297831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:15:22.80982336Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:15:22.94654957Z caller=checkpoint.go:372 msg="writing series" size="584 kB"
level=info ts=2026-05-13T14:15:23.292147879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:15:23.294906314Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:15:23.294924059Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:15:23.294946742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:15:23.297118449Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:15:25.249837417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:15:25.253614454Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:15:25.282680121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:15:25.286955543Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:15:25.286969156Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:15:25.286990167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:15:25.291139366Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:15:25.534807921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:15:25.538166682Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:15:25.7188682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:15:26.199146165Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-13T14:15:26.379053794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:27.791325738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:15:27.79433474Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:15:29.452562731Z caller=checkpoint.go:372 msg="writing series" size="633 kB"
level=debug ts=2026-05-13T14:15:29.452585529Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T14:15:30.264194549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:15:30.270370497Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T14:15:31.379221166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:32.296578579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:15:32.30167918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T14:15:32.705064839Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T14:15:33.605564015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:15:33.6082835Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:15:35.337024627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:15:35.341115674Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T14:15:35.958374542Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T14:15:36.378614203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:37.288087647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:15:37.291043443Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:15:37.291062506Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:15:37.291082462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:15:37.29332099Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:15:37.423846324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:15:37.429069287Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:15:37.654755119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:15:37.65983208Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:15:37.812212527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:15:37.814896582Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:15:39.121377636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:15:39.124394564Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:15:39.212145034Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=info ts=2026-05-13T14:15:41.379103403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:15:41.379130202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T14:15:41.379125053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:41.379138123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:15:41.818072594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:15:41.822510074Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:15:41.864438435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:15:41.867024067Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:15:41.943031203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:15:41.948688176Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:15:42.466661462Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-13T14:15:42.466726223Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=4
ts=2026-05-13T14:15:44.206983947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:15:44.236678511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:15:44.239435345Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:15:45.717814206Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-13T14:15:46.015691933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:15:46.019375977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T14:15:46.317952145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:60972"
level=warn ts=2026-05-13T14:15:46.379334239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:15:46.980470774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:15:46.983220928Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:15:48.970858019Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T14:15:49.148400423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:15:49.151235618Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T14:15:51.379293162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:15:52.224822816Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T14:15:52.224846115Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-13T14:15:52.562102005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:15:52.564909085Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:15:53.978116987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:15:53.979756053Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T14:15:55.310374254Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:15:55.475949278Z caller=checkpoint.go:372 msg="writing series" size="946 B"
level=warn ts=2026-05-13T14:15:56.378470968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:15:58.73081268Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T14:15:58.7308344Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T14:16:00.742402122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:16:00.745488847Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:16:00.973278842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:16:00.976612591Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=warn ts=2026-05-13T14:16:01.378679837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:16:01.982461423Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T14:16:02.838541035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:16:02.841840594Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:16:03.247091757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:16:03.250493036Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:16:04.154749981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:16:04.157961038Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:16:05.235807159Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-13T14:16:06.379066343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:07.874371104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:16:07.877869428Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:16:08.491125443Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T14:16:08.491150391Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=debug ts=2026-05-13T14:16:11.35904182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:16:11.360103767Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:16:11.378421059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:16:11.378440193Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:16:11.378448388Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:16:11.378440862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:11.663972574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:16:11.670834961Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:16:11.741748188Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
ts=2026-05-13T14:16:14.207809573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:16:14.994951247Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=info ts=2026-05-13T14:16:15.453087991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:16:15.460684415Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:16:16.37925313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:17.673200834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:16:17.679256164Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:16:18.247777706Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T14:16:19.351530251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:16:19.355198784Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:16:19.493589799Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:16:20.657926959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:16:20.660705847Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:16:20.660722079Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:16:20.660742811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:16:20.66283291Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T14:16:21.378634474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:16:21.500934826Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T14:16:23.10261722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:16:23.105984665Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:16:24.408611213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:16:24.41154748Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:16:24.753608495Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=info ts=2026-05-13T14:16:25.100380397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:16:25.103078061Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:16:26.373746768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:16:26.376352342Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:16:26.379412881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:26.394539215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:16:26.396825859Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:16:27.991017043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:16:27.99398808Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:16:28.008774239Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T14:16:28.00878751Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=info ts=2026-05-13T14:16:29.031771442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:16:29.035493599Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:16:30.419382038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:16:30.42321326Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T14:16:31.259907584Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=warn ts=2026-05-13T14:16:31.37932192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:32.710977441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:16:32.716378828Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:16:32.716397307Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:16:32.716422046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:16:32.720859424Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:16:33.08795938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:16:33.093674952Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:16:33.209078402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:16:33.212233554Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:16:34.512914157Z caller=checkpoint.go:372 msg="writing series" size="607 kB"
level=info ts=2026-05-13T14:16:35.678047222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:16:35.680737035Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:16:36.025592912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:16:36.028418746Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:16:36.045471396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:16:36.047808155Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T14:16:36.378875856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:16:37.766038046Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T14:16:38.923303266Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T14:16:41.018495826Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-13T14:16:41.378774208Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:16:41.378796825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:16:41.378804724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:16:41.378849048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:43.460012815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:16:43.463697668Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:16:43.463717614Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:16:43.463742269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:16:43.466682378Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=info ts=2026-05-13T14:16:43.814665999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:16:43.819911539Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T14:16:44.209201794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:16:44.272065641Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-13T14:16:44.272080233Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-13T14:16:46.378506148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:16:47.525304282Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=info ts=2026-05-13T14:16:48.400348919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:16:48.403800944Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:16:48.706516309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:16:48.709357995Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:16:50.839074313Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T14:16:50.839099286Z caller=checkpoint.go:384 msg="flushing series" totalSize="55 MB" series=2
level=warn ts=2026-05-13T14:16:51.378551303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:16:52.139503598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:16:52.142626975Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:16:54.031115079Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-13T14:16:54.860613498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:16:54.867026543Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:16:55.199044461Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T14:16:56.378609466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:16:57.284528482Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=info ts=2026-05-13T14:16:57.626186195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:16:57.629610568Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:16:57.866370926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:16:57.87191727Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:17:00.023882515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:17:00.028031317Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:17:00.028052227Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:17:00.028075556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:17:00.030987167Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:17:00.250747349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:17:00.25451885Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-13T14:17:00.504730452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:44744"
level=debug ts=2026-05-13T14:17:00.536714128Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-13T14:17:01.379217121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:17:03.456033134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:17:03.460350407Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:17:03.790259983Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-13T14:17:06.379433146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:07.0428218Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T14:17:08.335477817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:17:08.342033603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:17:09.885547659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:17:09.888539576Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:17:10.299061645Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-13T14:17:10.299090498Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=6
level=info ts=2026-05-13T14:17:11.321601455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:17:11.32519871Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:17:11.3595207Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:17:11.360572417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:17:11.378912166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:17:11.378926447Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:17:11.378933713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:17:11.378965197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:17:11.976286393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:17:11.979220247Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:17:12.184751821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:17:12.190571687Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:17:13.079017412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:17:13.082137197Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:17:13.548798795Z caller=checkpoint.go:372 msg="writing series" size="332 kB"
ts=2026-05-13T14:17:13.559534886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50002"
ts=2026-05-13T14:17:14.210845335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:17:15.557054318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:17:15.561240448Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T14:17:16.322730651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:55726"
level=warn ts=2026-05-13T14:17:16.379061975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:16.802236689Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T14:17:19.470013724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:17:19.47583784Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:17:19.478864971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:17:19.481344794Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:17:19.993862896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:17:19.996906623Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:17:19.996928843Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:17:19.996960982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:17:19.999286831Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:17:20.054781162Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-13T14:17:20.873900005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:17:20.879079573Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:17:20.879097508Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:17:20.879121788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:17:20.883469766Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=warn ts=2026-05-13T14:17:21.378817803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:21.748933768Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:17:22.020227026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:17:22.023569879Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:17:23.30875252Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T14:17:23.308774622Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T14:17:24.907344656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:17:24.910496524Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:17:25.062950627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:17:25.066312205Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=info ts=2026-05-13T14:17:25.588907225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:17:25.598225369Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:17:26.378716415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:26.561127523Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T14:17:26.561146202Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-13T14:17:28.471841176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:17:28.474823574Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:17:29.690098582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:17:29.693064191Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:17:29.825458985Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-13T14:17:29.825475939Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
ts=2026-05-13T14:17:30.505998783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42344"
level=warn ts=2026-05-13T14:17:31.378416409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:33.067228226Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-13T14:17:33.398074162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:17:33.401205729Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:17:33.972914334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:17:33.976304029Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:17:33.976325079Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:17:33.976348409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:17:33.978825749Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:17:36.320416394Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T14:17:36.378970395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:17:37.059854338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:17:37.064897374Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:17:38.118328259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:17:38.653093652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:17:38.659736492Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:17:38.748976176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:17:38.751760321Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:17:39.418925092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:17:39.421867967Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:17:39.575427595Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T14:17:39.575447113Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-13T14:17:41.379060703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:17:41.379078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:41.379085087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:17:41.379095424Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:17:42.826126053Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
ts=2026-05-13T14:17:44.212402142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:17:44.946006997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:17:44.949479102Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:17:46.078771896Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-13T14:17:46.324555774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:39736"
level=warn ts=2026-05-13T14:17:46.378786049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:17:47.479036548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:17:47.48316821Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:17:48.354873023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:17:48.358036424Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:17:49.331613697Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T14:17:49.469196713Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:17:50.175017116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:17:50.181870534Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:17:50.250085293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:17:50.253297363Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:17:50.253313299Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:17:50.253334901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:17:50.256285582Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T14:17:51.379234208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:17:52.585238356Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-13T14:17:55.838431987Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-13T14:17:56.378499126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:17:56.930301181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:17:56.936214619Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:17:59.091616873Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-13T14:18:00.47335627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:18:00.480457953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T14:18:01.378900127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:01.96267991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:18:01.965879698Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:18:02.344192158Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=debug ts=2026-05-13T14:18:02.344208068Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-13T14:18:03.839479686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:18:03.845395834Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:18:03.861603721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:18:03.864211451Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:18:03.864227607Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:18:03.864245829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:18:03.866769785Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:18:05.19044511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:18:05.193653373Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:18:05.443402234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:18:05.447295689Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:18:05.596523818Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T14:18:06.262521604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:18:06.266606667Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:18:06.266625681Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:18:06.266649543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:18:06.27002238Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T14:18:06.379228716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:07.828448312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:18:07.83179782Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:18:08.502617059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:18:08.505586743Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:18:08.526636283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:18:08.530293041Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:18:08.727853342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:18:08.730608503Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:18:08.850081001Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=info ts=2026-05-13T14:18:09.40264071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:18:09.405352053Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:18:11.335942887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:18:11.338902262Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T14:18:11.359569642Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:18:11.360644599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T14:18:11.379029596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:11.379034361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:18:11.379052818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:18:11.379063729Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:18:12.102715265Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
level=info ts=2026-05-13T14:18:12.819734548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:18:12.822457393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T14:18:14.214238196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:18:14.312256746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:18:14.701525908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:18:14.704323326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:18:15.356305609Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-05-13T14:18:15.895564605Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T14:18:16.326117234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:59976"
level=info ts=2026-05-13T14:18:16.356914368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="402 B" avg_comp="402 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:18:16.359623932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:18:16.378837076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:18:18.609024705Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T14:18:19.251773358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:18:19.255458411Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:18:19.255478128Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:18:19.255501114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:18:19.25867567Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:18:21.252280702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:18:21.257706187Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T14:18:21.37878424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:18:21.861773234Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T14:18:22.207709533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:18:22.210793507Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:18:22.210812839Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:18:22.210835152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:18:22.213198126Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:18:22.552199663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:18:22.555217479Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:18:22.555239561Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:18:22.555265895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:18:22.557726868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:18:25.11527365Z caller=checkpoint.go:372 msg="writing series" size="666 kB"
level=warn ts=2026-05-13T14:18:26.37908182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:18:28.368401792Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-13T14:18:30.508691595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:42554"
level=info ts=2026-05-13T14:18:31.138785329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:18:31.143976Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T14:18:31.379382637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:18:31.62125336Z caller=checkpoint.go:372 msg="writing series" size="476 B"
level=info ts=2026-05-13T14:18:32.874470759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:18:32.877637983Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:18:34.375786359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:18:34.379614138Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T14:18:34.874461757Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=debug ts=2026-05-13T14:18:34.874482631Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=10
level=warn ts=2026-05-13T14:18:36.378503537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:36.571303112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:18:36.578538263Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:18:38.127400532Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-13T14:18:38.425486256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:18:38.431332434Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:18:39.141187889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:18:39.144742796Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:18:39.693422722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:18:39.696269257Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:18:40.548668656Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:18:41.019004053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:18:41.022502914Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:18:41.378674399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:18:41.378703444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T14:18:41.378697064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:41.378712721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:18:41.379772929Z caller=checkpoint.go:384 msg="flushing series" totalSize="202 kB" series=1
level=info ts=2026-05-13T14:18:41.384466987Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-13T14:18:41.385655868Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006135876s
level=info ts=2026-05-13T14:18:42.708219097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:18:42.711130589Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:18:44.048856911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:18:44.052898705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T14:18:44.21552203Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-bnzkg-fe4a6947 10.128.0.55:7946"
level=info ts=2026-05-13T14:18:44.225209191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:18:44.232316539Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:18:45.248311402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:18:45.251450825Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:18:45.251468858Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:18:45.251488442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:18:45.253937651Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
ts=2026-05-13T14:18:46.327694112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:58144"
level=warn ts=2026-05-13T14:18:46.379188502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:49.801443584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:18:49.804162533Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:18:49.820662915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:18:50.682286575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:18:50.685917336Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T14:18:50.952675358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:18:50.956380061Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=warn ts=2026-05-13T14:18:51.378738233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:54.243300197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:18:54.249992542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T14:18:56.378759318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:18:56.724352617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:18:56.727222015Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:18:59.004499589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:18:59.007605789Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:18:59.45413573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:18:59.456971906Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:18:59.791746696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:18:59.794619546Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:19:00.398347629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:19:00.403769521Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
ts=2026-05-13T14:19:00.509877952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:59528"
level=info ts=2026-05-13T14:19:00.876411623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:19:00.87942267Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:19:01.097925214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:19:01.100728494Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:19:01.100746691Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:19:01.100775895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:19:01.104366218Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T14:19:01.378445158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:04.831119553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:19:05.504976324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:19:05.508776558Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:19:06.378762536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:19:08.142229064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:19:08.146264384Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:19:11.338562493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:19:11.341520728Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T14:19:11.359628635Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T14:19:11.359910831Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:19:11.37926615Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T14:19:11.379294949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:19:11.379322045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T14:19:11.379321263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:19:11.37933105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T14:19:11.379398957Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-13T14:19:11.379406716Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T14:19:11.379423166Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.054µs success=true
level=debug ts=2026-05-13T14:19:11.379592411Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T14:19:13.4691394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:19:13.472580707Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:19:13.808443717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:19:13.811064229Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:19:14.051902392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:19:14.054594711Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:19:14.199983628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:19:14.202455918Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T14:19:14.216661091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:19:14.633610237Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=warn ts=2026-05-13T14:19:16.379090557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:17.886781909Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T14:19:17.886806382Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-13T14:19:20.685209828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:19:20.688127214Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:19:21.139276717Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-13T14:19:21.378929481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:19:23.170018485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:19:23.174921215Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:19:24.392529143Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-13T14:19:25.959737475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:19:25.965536647Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T14:19:26.37873065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:19:26.791935999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:19:26.79564216Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:19:27.645444042Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-13T14:19:28.438604888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:19:28.442672007Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T14:19:30.511546641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.54:48040"
level=debug ts=2026-05-13T14:19:30.898091731Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T14:19:31.379234232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:31.509695676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:19:31.509741443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2ab140b022d7493d immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="189 kB" avg_uncomp="189 kB" idle=1 labels="{stream=\"build-e2e-uaem\"}"
level=error ts=2026-05-13T14:19:31.513151651Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2ab140b022d7493d/19e2168893e%3A19e217d8edc%3A80c1fd73?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uaem\"}"
level=info ts=2026-05-13T14:19:31.8229366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:19:31.826093842Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=info ts=2026-05-13T14:19:33.015165053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:19:33.017910022Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:19:33.472294128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:19:33.475759781Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:19:33.475777812Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:19:33.47580067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:19:33.47863435Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:19:34.151579973Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T14:19:34.864908805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:19:34.86838476Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:19:35.970970615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:19:35.97728818Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T14:19:36.379355156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:37.404186564Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-13T14:19:38.64465291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:19:38.648434435Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:19:39.820430134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:19:39.826669436Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:19:39.826689426Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:19:39.826714113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:19:39.832360477Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:19:39.84242549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:19:39.844783204Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:19:40.30492612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:19:40.3108814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:19:40.657092786Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-13T14:19:41.378972948Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:19:41.378998002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:19:41.379006251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:19:41.379018169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:19:41.915122202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:19:41.918463674Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:19:43.044311994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:19:43.047224094Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T14:19:43.567078387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:50766"
level=debug ts=2026-05-13T14:19:43.91061603Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
ts=2026-05-13T14:19:44.218075745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T14:19:44.742883972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:19:44.747915099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=info ts=2026-05-13T14:19:45.141924859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2ab140b022d7493d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-uaem\"}"
level=error ts=2026-05-13T14:19:45.144733567Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2ab140b022d7493d/19e2168893e%3A19e217d8edc%3A80c1fd73?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uaem\"}"
level=debug ts=2026-05-13T14:19:45.980326077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T14:19:45.981943132Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T14:19:46.28873323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:19:46.291601116Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:19:46.29161849Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:19:46.291639089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:19:46.293851726Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=warn ts=2026-05-13T14:19:46.379334594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:47.163765365Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-13T14:19:47.293623293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:19:47.297147947Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:19:50.415954541Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T14:19:50.415978139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-05-13T14:19:51.379213727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:53.669092489Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T14:19:55.925053149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:19:55.93133102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:19:56.225313475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:19:56.22880432Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:19:56.277004618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:19:56.279761169Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T14:19:56.379027655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:19:56.922719532Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=debug ts=2026-05-13T14:19:57.997844508Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:20:00.17576585Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=warn ts=2026-05-13T14:20:01.379311161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:02.868793166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:20:02.872016341Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:20:03.042406332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:20:03.045501653Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=debug ts=2026-05-13T14:20:03.428762637Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T14:20:05.371225614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:20:05.374926487Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T14:20:06.379213821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:06.556097424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:20:06.561468656Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:20:06.681753189Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T14:20:09.43965793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:20:09.44429655Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:20:09.93521522Z caller=checkpoint.go:372 msg="writing series" size="970 kB"
level=debug ts=2026-05-13T14:20:09.935237131Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-13T14:20:10.238550576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:20:10.241466246Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:20:10.241488078Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:20:10.241510217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:20:10.243681794Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:20:11.359258213Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:20:11.360308858Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:20:11.378658925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:20:11.378674707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:20:11.378687608Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:20:11.378695282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:12.673054222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:20:12.675927994Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T14:20:13.187324109Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T14:20:14.219414277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=info ts=2026-05-13T14:20:14.229088942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:20:14.232872522Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
ts=2026-05-13T14:20:16.33141917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:36904"
level=warn ts=2026-05-13T14:20:16.378838703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:20:16.440024403Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T14:20:16.866161443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:20:16.869120662Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:20:18.281199221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:20:18.284754011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:20:19.693431114Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T14:20:19.772763421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:20:19.776725266Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T14:20:19.909989156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2ab140b022d7493d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-uaem\"}"
level=error ts=2026-05-13T14:20:19.912931636Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2ab140b022d7493d/19e2168893e%3A19e217d8edc%3A80c1fd73?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uaem\"}"
level=info ts=2026-05-13T14:20:19.995208161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:20:19.998327292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=warn ts=2026-05-13T14:20:21.378591568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:22.303450621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:20:22.306704844Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:20:22.946781344Z caller=checkpoint.go:372 msg="writing series" size="584 kB"
level=info ts=2026-05-13T14:20:23.859762609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:20:23.862700832Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T14:20:24.125232513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:20:24.135324443Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T14:20:24.710057711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:20:24.715374557Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:20:25.697666608Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T14:20:26.199138578Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-13T14:20:26.378752035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:29.386358167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:20:29.389237565Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T14:20:29.452746416Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=debug ts=2026-05-13T14:20:29.452759044Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-13T14:20:31.124782027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:20:31.127516151Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T14:20:31.378794617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:31.469077183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:20:31.471946007Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:20:31.773986362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:20:31.776691225Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:20:32.705896315Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T14:20:34.609487378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:20:34.616223868Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:20:34.616244777Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:20:34.616271839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:20:34.623024181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:20:34.905805267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:20:34.908816932Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:20:35.52863726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:20:35.535095142Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:20:35.958492729Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T14:20:36.379048164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:37.861452923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:20:37.868430791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:20:39.211738133Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-13T14:20:40.797708102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:20:41.379077904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:20:41.379102063Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:20:41.379110862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:20:41.379142443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:20:42.465958007Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-13T14:20:42.465980981Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=4
ts=2026-05-13T14:20:44.220346893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:20:45.717250161Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-13T14:20:45.756504045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:20:45.759600566Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T14:20:46.332457363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:33118"
level=warn ts=2026-05-13T14:20:46.379072788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:20:47.788056822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:20:47.791328882Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:20:47.841399694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:20:47.847723954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T14:20:48.970093687Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T14:20:51.378937154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:20:52.224322832Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T14:20:52.224344239Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-13T14:20:53.142412497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:20:53.146242047Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:20:53.871472501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:20:53.874489104Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=debug ts=2026-05-13T14:20:54.062194672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:20:55.172432009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:20:55.1759515Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T14:20:55.476875747Z caller=checkpoint.go:372 msg="writing series" size="946 B"
level=warn ts=2026-05-13T14:20:56.37914108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:20:58.730560382Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T14:20:58.73058263Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T14:21:01.379204028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:01.785098509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:21:01.789075172Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T14:21:01.86428423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:21:01.867728567Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T14:21:01.982444091Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=info ts=2026-05-13T14:21:04.651629545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:21:04.657000812Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:21:05.235478968Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-13T14:21:06.379100622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:07.560950539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:21:07.56418973Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T14:21:08.1917554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2ab140b022d7493d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-uaem\"}"
level=error ts=2026-05-13T14:21:08.195248026Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2ab140b022d7493d/19e2168893e%3A19e217d8edc%3A80c1fd73?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uaem\"}"
level=debug ts=2026-05-13T14:21:08.493167056Z caller=checkpoint.go:372 msg="writing series" size="4.8 MB"
level=debug ts=2026-05-13T14:21:08.493205036Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.4 MB" series=3
level=info ts=2026-05-13T14:21:09.628197166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:21:09.632095373Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=debug ts=2026-05-13T14:21:11.359574451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:21:11.360622442Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:21:11.378960478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:21:11.378974517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:21:11.378982565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:21:11.378983632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:21:11.741250451Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-05-13T14:21:11.845163793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:21:11.848400876Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:21:12.078074377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:21:12.081414239Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:21:14.157276912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:21:14.160829718Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:21:14.160856028Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:21:14.160880916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:21:14.163045495Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T14:21:14.221307939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:21:14.287045151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:21:14.29423339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:21:14.701554091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:21:14.704314197Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:21:14.994467789Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=info ts=2026-05-13T14:21:15.49352526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:21:15.497441994Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T14:21:16.333616794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:35232"
level=warn ts=2026-05-13T14:21:16.378904067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:17.362744284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:21:17.365660889Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T14:21:17.560126361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:21:17.562899977Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:21:18.247496242Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T14:21:21.379420939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:21:21.500733036Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T14:21:22.116577438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:21:22.119556414Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=debug ts=2026-05-13T14:21:22.753443642Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:21:24.354677448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:21:24.35998901Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:21:24.754332998Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-13T14:21:25.737680439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:21:25.740512701Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T14:21:26.378853598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:26.607052498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:21:26.609742754Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:21:27.280379269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:21:27.283506348Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:21:28.009391171Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T14:21:28.009419544Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=info ts=2026-05-13T14:21:28.463221136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:21:28.46591486Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:21:30.138018222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:21:30.140668528Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:21:31.166719136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:21:31.172664453Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T14:21:31.260254508Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-13T14:21:31.379230823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:21:34.513349806Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=debug ts=2026-05-13T14:21:35.13255991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:21:36.045011839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:21:36.048306126Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=warn ts=2026-05-13T14:21:36.378966622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:36.911065123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:21:36.91595188Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:21:37.765333426Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-13T14:21:41.018566258Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-13T14:21:41.378401149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:41.378417656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:21:41.378439252Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:21:41.37844696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T14:21:44.222867023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
level=debug ts=2026-05-13T14:21:44.271745879Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-13T14:21:45.70578682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:21:45.711344828Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T14:21:46.378923314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:46.49621715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:21:46.499649081Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:21:46.5157193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:21:46.518087748Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:21:46.518104393Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:21:46.518123942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:21:46.520152524Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T14:21:47.525229483Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=debug ts=2026-05-13T14:21:47.525250904Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-13T14:21:48.18772947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:21:48.1905895Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T14:21:48.600976156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T14:21:48.703138401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:21:48.706447677Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T14:21:50.839049998Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T14:21:50.839074246Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=1
level=warn ts=2026-05-13T14:21:51.379246976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:21:51.715625576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:21:51.720018014Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=info ts=2026-05-13T14:21:51.868227658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:21:51.872576274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T14:21:53.674091439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:21:53.676936812Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:21:54.030831161Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-13T14:21:55.771881054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:21:55.776095069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T14:21:56.378424765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:21:57.284239775Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=info ts=2026-05-13T14:21:59.439848747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:21:59.446294465Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:21:59.703962223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:21:59.707449223Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=debug ts=2026-05-13T14:22:00.536742048Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-13T14:22:01.378688464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:03.051632469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:22:03.05450378Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:22:03.789775679Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T14:22:04.080589623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2ab140b022d7493d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-uaem\"}"
level=error ts=2026-05-13T14:22:04.083519494Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2ab140b022d7493d/19e2168893e%3A19e217d8edc%3A80c1fd73?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uaem\"}"
level=info ts=2026-05-13T14:22:04.453989879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:22:04.459571456Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:22:04.459595289Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:22:04.459622997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:22:04.465203908Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T14:22:05.084923705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:22:05.087706389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:22:05.466913305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:22:05.469812721Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T14:22:06.379150061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:22:07.042689751Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T14:22:10.298813287Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T14:22:10.29883531Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=6
level=info ts=2026-05-13T14:22:10.55235225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:22:10.555583859Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:22:11.197221799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:22:11.200192288Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:22:11.200209639Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:22:11.200229451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:22:11.202473571Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:22:11.277867869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:22:11.280830417Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T14:22:11.359553935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:22:11.360610476Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:22:11.378976279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:22:11.378995751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:22:11.379003225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:22:11.379020292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:11.504686096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:22:11.507813619Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=info ts=2026-05-13T14:22:12.881941941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:22:12.884950642Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T14:22:13.548856735Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
ts=2026-05-13T14:22:13.574277592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.55:34082"
ts=2026-05-13T14:22:14.223916588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-k75jh-b82989bd 10.128.0.54:7946"
ts=2026-05-13T14:22:16.336747407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:41434"
level=warn ts=2026-05-13T14:22:16.378831615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:16.523129359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:22:16.527159816Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T14:22:16.802291433Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T14:22:17.315357246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:22:17.318392802Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T14:22:18.298213018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=815f37b21d040dff immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"tekton-results\"}"
level=error ts=2026-05-13T14:22:18.3039279Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/815f37b21d040dff/19e215e24c8%3A19e21815f6c%3Ab8ea7435?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tekton-results\"}"
level=debug ts=2026-05-13T14:22:20.055288801Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=debug ts=2026-05-13T14:22:20.608852636Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T14:22:21.379407293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:22.507900193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:22:22.512550929Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:22:23.308072284Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-13T14:22:23.308094679Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-13T14:22:23.787863808Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T14:22:24.689088576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:22:24.692096521Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
level=info ts=2026-05-13T14:22:24.831431442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:22:24.833988911Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=warn ts=2026-05-13T14:22:26.379131787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:22:26.562209934Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T14:22:26.562229029Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-13T14:22:27.991621271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="436 kB" avg_comp="436 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T14:22:27.995763708Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e215db99f%3A19e21ab2560%3Aa044b8b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T14:22:29.74154746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:22:29.747511833Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:22:29.747531412Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:22:29.747552565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:22:29.752119513Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T14:22:29.826710339Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-13T14:22:29.826727271Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-05-13T14:22:30.21689239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.4 MB" avg_comp="1.7 MB" total_uncomp="22 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T14:22:30.22414977Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e215db6b2%3A19e2179b294%3A664642ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T14:22:30.821761436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T14:22:30.825169254Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e215dbb43%3A19e21a4f0e4%3Af815a5e7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T14:22:31.378639987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:22:33.066907014Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T14:22:34.88643684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T14:22:34.889376241Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e215e24c8%3A19e21744bc4%3Ae8b4f40b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T14:22:35.732405631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="646 kB" avg_comp="646 kB" total_uncomp="3.4 MB" avg_uncomp="3.4 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T14:22:35.736633432Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e215db9b0%3A19e21a50085%3A93bd18f7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T14:22:36.32053691Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T14:22:36.379470465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:36.418506007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="512 kB" avg_comp="512 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T14:22:36.422423419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e215dbd3d%3A19e21793024%3A785895f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T14:22:39.57449312Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T14:22:39.574519504Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=info ts=2026-05-13T14:22:41.37874471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T14:22:41.378758067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:22:41.378777571Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:22:41.378786063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T14:22:42.826043032Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
ts=2026-05-13T14:22:44.225263517Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=info ts=2026-05-13T14:22:45.611984315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T14:22:45.617836564Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e215dbd2c%3A19e21921dcf%3Ae9c48cfa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T14:22:46.079288812Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-13T14:22:46.378650903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:46.787431658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="34 kB" avg_comp="34 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T14:22:46.792294329Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e215dcae6%3A19e2160ba6c%3Ac24d410b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T14:22:48.942929803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T14:22:48.948695023Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e215e24c8%3A19e2184e1dc%3A94192d4b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T14:22:49.286689057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T14:22:49.289674105Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e215e4b16%3A19e215f242c%3Ac6889476?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T14:22:49.331797182Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-13T14:22:49.898450791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T14:22:49.901611711Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e215e3864%3A19e215e4b17%3A29c0c41b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T14:22:50.808418465Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T14:22:51.378974193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:22:51.550618527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=54765354a87288d0 immediate=false num_chunks=1 total_comp="401 kB" avg_comp="401 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:22:51.554261621Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/54765354a87288d0/19e21683603%3A19e2175b83f%3Adadf7d64?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-jvgz\"}"
level=error ts=2026-05-13T14:22:51.554278911Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:22:51.554303395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T14:22:51.557594443Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e215e8578%3A19e21b12db4%3Ae234f696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T14:22:52.585475861Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-13T14:22:53.175167499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e60954acd55025ec immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-lber\"}"
level=error ts=2026-05-13T14:22:53.178768121Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e60954acd55025ec/19e2169b4d3%3A19e21754ec7%3Afc26c908?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-lber\"}"
level=info ts=2026-05-13T14:22:54.040027665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T14:22:54.042964206Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e215e24c8%3A19e215e6c94%3A142e85e0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T14:22:54.712830079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.4 MB" total_uncomp="132 MB" avg_uncomp="22 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T14:22:54.719817671Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e215dc257%3A19e215ed4e3%3A38c621d4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T14:22:54.75500672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=44842b72361feba3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="43 kB" avg_uncomp="43 kB" full=1 labels="{stream=\"konflux-poai-managed\"}"
level=error ts=2026-05-13T14:22:54.757611922Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/44842b72361feba3/19e2174c8c3%3A19e217808cc%3A74f9cc86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-poai-managed\"}"
level=debug ts=2026-05-13T14:22:55.838256881Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-13T14:22:56.07507445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.3 kB" avg_comp="6.3 kB" total_uncomp="30 kB" avg_uncomp="30 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T14:22:56.078106942Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e215dbc42%3A19e215e3865%3A13dae99f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T14:22:56.379468216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:22:59.091366593Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-13T14:23:00.105392092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8593a4656f853a89 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-gndb\"}"
level=error ts=2026-05-13T14:23:00.108452401Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8593a4656f853a89/19e216795db%3A19e2167b134%3A87d15f22?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-gndb\"}"
level=info ts=2026-05-13T14:23:01.054142326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="420 B" avg_comp="420 B" total_uncomp="344 B" avg_uncomp="344 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T14:23:01.057327057Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2161350b%3A19e2161350c%3Af1764f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T14:23:01.378707916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T14:23:01.904285507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="3.7 MB" avg_uncomp="3.7 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T14:23:01.907844824Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e215e8578%3A19e21b12db4%3Ae234f696?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T14:23:02.343749818Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=info ts=2026-05-13T14:23:03.257453671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2ab140b022d7493d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-uaem\"}"
level=error ts=2026-05-13T14:23:03.260407263Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/2ab140b022d7493d/19e2168893e%3A19e217d8edc%3A80c1fd73?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uaem\"}"
level=debug ts=2026-05-13T14:23:05.596979058Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T14:23:06.032142603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T14:23:06.032155911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="288 kB" avg_uncomp="288 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T14:23:06.03513733Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e215e24c8%3A19e215e4b43%3Ae0317d19?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T14:23:06.035141375Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e215dbd4d%3A19e215ffdbf%3A1ac0cd72?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T14:23:06.379153566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:23:08.850413054Z caller=checkpoint.go:372 msg="writing series" size="818 B"
level=debug ts=2026-05-13T14:23:11.359391524Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.63:11211 count=1
level=info ts=2026-05-13T14:23:11.36044498Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T14:23:11.378797778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T14:23:11.378813845Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T14:23:11.378822264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T14:23:11.378822561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T14:23:12.103026306Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-13T14:23:12.191362549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:23:12.195640077Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:23:12.195661677Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T14:23:12.195697537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T14:23:12.19874016Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e2167dd5e%3A19e21693004%3Ad364f1ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T14:23:14.226621572Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-177c7b43 10.128.0.57:7946"
level=debug ts=2026-05-13T14:23:15.355843579Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-05-13T14:23:15.355865032Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-05-13T14:23:15.692719695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13a709de36c4e285 immediate=false num_chunks=1 total_comp="200 kB" avg_comp="200 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"stat-rep-byci\"}"
level=error ts=2026-05-13T14:23:15.695881872Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/13a709de36c4e285/19e21691893%3A19e2170e70c%3Aa1b4e124?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-byci\"}"
ts=2026-05-13T14:23:16.339738273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.57:48474"
level=warn ts=2026-05-13T14:23:16.379066803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"