---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  creationTimestamp: "2026-10-23T02:44:32Z"
  generation: 1
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/name: trainer
  managedFields:
  - apiVersion: monitoring.coreos.com/v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:labels:
          f:app.kubernetes.io/component: {}
          f:app.kubernetes.io/name: {}
      f:spec:
        f:groups:
          k:{"name":"rhoai.trainer.telemetry"}:
            .: {}
            f:name: {}
            f:rules: {}
    manager: kubectl
    operation: Apply
    time: "2026-10-23T02:44:32Z"
  name: kubeflow-trainer-telemetry
  namespace: redhat-ods-applications
  resourceVersion: "11364"
  uid: 1f786903-0e5a-4d9b-96db-6d605ce77f2a
spec:
  groups:
  - name: rhoai.trainer.telemetry
    rules:
    - expr: |
        sum(increase(kubeflow_trainer_trainjob_created_total[24h]))
      record: cluster:rhoai_kubeflow_trainer_trainjob_created:increase24hr
    - expr: |
        label_replace(
          sum by (le) (
            label_replace(
              increase(
                kubeflow_trainer_trainjob_requested_training_nodes_per_job_bucket{
                  runtime_ref=~"^trainer[.]kubeflow[.]org/ClusterTrainingRuntime//(torch-distributed|torch-distributed-with-cache|torch-distributed-cpu|torch-distributed-rocm|torch-distributed-cuda130-torch211-py312|torch-distributed-rocm714-torch211-py312|torch-distributed-cpu-torch211-py312)$",
                  le=~"^(4(\\.0)?|[+]Inf)$"
                }[24h]
              ),
              "le", "4.0", "le", "^4(\\.0)?$"
            )
          ),
          "runtime_ref", "torch-distributed", "le", ".*"
        )
        or label_replace(
          sum by (le) (
            label_replace(
              increase(
                kubeflow_trainer_trainjob_requested_training_nodes_per_job_bucket{
                  runtime_ref=~"^trainer[.]kubeflow[.]org/ClusterTrainingRuntime//(training-hub|training-hub-cpu|training-hub-rocm|training-hub-th09-cpu-torch211-py312|training-hub-th09-cuda130-torch211-py312|training-hub-th09-rocm714-torch211-py312)$",
                  le=~"^(4(\\.0)?|[+]Inf)$"
                }[24h]
              ),
              "le", "4.0", "le", "^4(\\.0)?$"
            )
          ),
          "runtime_ref", "training-hub", "le", ".*"
        )
        or label_replace(
          sum by (le) (
            label_replace(
              increase(
                kubeflow_trainer_trainjob_requested_training_nodes_per_job_bucket{
                  runtime_ref=~"^trainer[.]kubeflow[.]org/ClusterTrainingRuntime//(openmpi-cpu|openmpi-cuda)$",
                  le=~"^(4(\\.0)?|[+]Inf)$"
                }[24h]
              ),
              "le", "4.0", "le", "^4(\\.0)?$"
            )
          ),
          "runtime_ref", "openmpi", "le", ".*"
        )
      record: runtime:rhoai_kubeflow_trainer_trainjob_requested_training_nodes:increase24hr
    - expr: |
        sum by (le) (
          label_replace(
            label_replace(
              increase(
                kubeflow_trainer_trainjob_lifetime_seconds_bucket{
                  le=~"^(43200(\\.0)?|345600(\\.0)?|[+]Inf)$"
                }[24h]
              ),
              "le", "43200.0", "le", "^43200(\\.0)?$"
            ),
            "le", "345600.0", "le", "^345600(\\.0)?$"
          )
        )
      record: cluster:rhoai_kubeflow_trainer_trainjob_lifetime_seconds:increase24hr