go test ./tests/e2e/ -v -timeout 120m -count=1 -run ^TestMonitoring$ -timeout 90m -api-mode module -monitoring-cr-name default-monitoring -install-operators=true -olm-timeout 15m -eventually-timeout 10m go: downloading github.com/onsi/gomega v1.44.0 go: downloading github.com/opendatahub-io/odh-platform-utilities v0.5.0 go: downloading github.com/openshift/api v0.0.0-20261002115817-f8795cdde518 go: downloading github.com/openshift/library-go v0.0.0-20261001144932-192662696fc7 go: downloading github.com/stretchr/testify v1.12.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading k8s.io/api v0.37.1 go: downloading k8s.io/apimachinery v0.37.1 go: downloading k8s.io/client-go v0.37.1 go: downloading sigs.k8s.io/controller-runtime v0.25.2 go: downloading github.com/itchyny/gojq v0.12.19 go: downloading github.com/google/go-cmp v0.7.0 go: downloading k8s.io/apiserver v0.37.1 go: downloading k8s.io/klog/v2 v2.140.0 go: downloading k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad go: downloading sigs.k8s.io/structured-merge-diff/v6 v6.4.2 go: downloading sigs.k8s.io/randfill v1.0.0 go: downloading k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 go: downloading sigs.k8s.io/yaml v1.6.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading golang.org/x/net v0.58.0 go: downloading github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 go: downloading k8s.io/streaming v0.37.1 go: downloading github.com/evanphx/json-patch/v5 v5.9.11 go: downloading github.com/go-logr/logr v1.4.4 go: downloading github.com/prometheus/client_golang v1.24.0 go: downloading go.yaml.in/yaml/v3 v3.0.5 go: downloading gopkg.in/evanphx/json-patch.v4 v4.13.0 go: downloading github.com/go-logr/zapr v1.3.0 go: downloading go.uber.org/zap v1.27.1 go: downloading sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 go: downloading github.com/itchyny/timefmt-go v0.1.8 go: downloading go.yaml.in/yaml/v2 v2.4.4 go: downloading github.com/google/gnostic-models v0.7.1 go: downloading google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af go: downloading golang.org/x/time v0.15.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/fxamacker/cbor/v2 v2.9.1 go: downloading golang.org/x/term v0.46.0 go: downloading golang.org/x/oauth2 v0.36.0 go: downloading github.com/moby/spdystream v0.5.1 go: downloading github.com/spf13/pflag v1.0.10 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/k8s-manifest-kit/engine v0.2.1-0.20260302092700-39c16f95d249 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.2 go: downloading github.com/prometheus/common v0.70.0 go: downloading github.com/prometheus/procfs v0.21.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading golang.org/x/text v0.42.0 go: downloading golang.org/x/sys v0.48.0 go: downloading github.com/x448/float16 v0.8.4 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee go: downloading github.com/go-openapi/jsonreference v1.0.0 go: downloading github.com/go-openapi/swag v0.27.1 go: downloading github.com/k8s-manifest-kit/pkg v0.2.1-0.20260302085906-6946a2e8e1d5 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/go-openapi/jsonpointer v1.0.0 go: downloading github.com/go-openapi/swag/cmdutils v0.27.1 go: downloading github.com/go-openapi/swag/conv v0.27.1 go: downloading github.com/go-openapi/swag/fileutils v0.27.1 go: downloading github.com/go-openapi/swag/jsonutils v0.27.1 go: downloading github.com/go-openapi/swag/loading v0.27.1 go: downloading github.com/go-openapi/swag/mangling v0.27.1 go: downloading github.com/go-openapi/swag/netutils v0.27.1 go: downloading github.com/go-openapi/swag/stringutils v0.27.1 go: downloading github.com/go-openapi/swag/typeutils v0.27.1 go: downloading github.com/go-openapi/swag/yamlutils v0.27.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/emicklei/go-restful/v3 v3.13.0 go: downloading github.com/go-openapi/swag/pools v0.27.1 === RUN TestMonitoring e2e_test.go:38: detected 3 schedulable nodes — expecting 2 replicas === RUN TestMonitoring/openshift-cert-manager-operator === NAME TestMonitoring test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /cert-manager-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=true, err=) test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription cert-manager-operator/openshift-cert-manager-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 1: patching operators.coreos.com/v1alpha1, Kind=Subscription cert-manager-operator/openshift-cert-manager-operator (resourceVersion="10959") test_context_test.go:413: [TestMonitoring] resource attempt 1: patch completed in 21ms (resourceVersion="10959") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=ClusterRole /odh-observability-opentelemetry-cert-manager test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=clusterroles.rbac.authorization.k8s.io "odh-observability-opentelemetry-cert-manager" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=ClusterRole /odh-observability-opentelemetry-cert-manager test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="11358") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding /odh-observability-opentelemetry-cert-manager test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 16ms (found=false, err=clusterrolebindings.rbac.authorization.k8s.io "odh-observability-opentelemetry-cert-manager" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding /odh-observability-opentelemetry-cert-manager test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 21ms (resourceVersion="11359") === RUN TestMonitoring/install-dependent-operators === RUN TestMonitoring/install-dependent-operators/cluster-observability-operator === PAUSE TestMonitoring/install-dependent-operators/cluster-observability-operator === RUN TestMonitoring/install-dependent-operators/tempo-product === PAUSE TestMonitoring/install-dependent-operators/tempo-product === RUN TestMonitoring/install-dependent-operators/opentelemetry-product === PAUSE TestMonitoring/install-dependent-operators/opentelemetry-product === CONT TestMonitoring/install-dependent-operators/cluster-observability-operator === CONT TestMonitoring/install-dependent-operators/opentelemetry-product === NAME TestMonitoring test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-cluster-observability-operator === CONT TestMonitoring/install-dependent-operators/tempo-product === NAME TestMonitoring test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-tempo-operator test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-opentelemetry-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=namespaces "openshift-tempo-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-tempo-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=namespaces "openshift-cluster-observability-operator" not found) test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=namespaces "openshift-opentelemetry-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-opentelemetry-operator test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-cluster-observability-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="11360") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="11361") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="11362") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=operatorgroups.operators.coreos.com "cluster-observability-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=operatorgroups.operators.coreos.com "tempo-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=operatorgroups.operators.coreos.com "opentelemetry-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 21ms (resourceVersion="11419") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 22ms (resourceVersion="11417") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 21ms (resourceVersion="11418") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=subscriptions.operators.coreos.com "cluster-observability-operator" not found) test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=subscriptions.operators.coreos.com "opentelemetry-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=subscriptions.operators.coreos.com "tempo-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 22ms (resourceVersion="11439") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 22ms (resourceVersion="11440") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 22ms (resourceVersion="11441") test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11507") test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 18ms (found=true, err=) test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11482") test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11465") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 20ms (resourceVersion="11507") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 20ms (resourceVersion="11482") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 20ms (resourceVersion="11465") test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 21ms (resourceVersion="11541") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 21ms (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 21ms (resourceVersion="11514") test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 19ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 19ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 21ms (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 20ms (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 20ms (resourceVersion="11541") test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 18ms (found=true, err=) test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 20ms (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 20ms (resourceVersion="11541") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 21ms (resourceVersion="11548") test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11514") test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 18ms (found=true, err=) test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 21ms (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 21ms (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 21ms (resourceVersion="11541") test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11514") test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 20ms (resourceVersion="11514") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 21ms (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 20ms (resourceVersion="11541") test_context_test.go:367: [TestMonitoring] resource attempt 8: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 8: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 8: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 8: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 8: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:371: [TestMonitoring] resource attempt 8: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 8: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="11628") test_context_test.go:371: [TestMonitoring] resource attempt 8: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 8: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 8: patch completed in 22ms (resourceVersion="11541") test_context_test.go:413: [TestMonitoring] resource attempt 8: patch completed in 21ms (resourceVersion="11628") test_context_test.go:413: [TestMonitoring] resource attempt 8: patch completed in 21ms (resourceVersion="11548") test_context_test.go:367: [TestMonitoring] resource attempt 9: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 9: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 9: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 9: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:371: [TestMonitoring] resource attempt 9: get completed in 19ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 9: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11541") test_context_test.go:413: [TestMonitoring] resource attempt 9: patch completed in 22ms (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 9: patch completed in 22ms (resourceVersion="11541") test_context_test.go:367: [TestMonitoring] resource attempt 10: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 10: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 10: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 10: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:371: [TestMonitoring] resource attempt 10: get completed in 19ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 10: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="11899") test_context_test.go:413: [TestMonitoring] resource attempt 10: patch completed in 23ms (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 10: patch completed in 23ms (resourceVersion="11899") test_context_test.go:367: [TestMonitoring] resource attempt 11: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 11: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 11: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 11: patch completed in 20ms (resourceVersion="11548") test_context_test.go:367: [TestMonitoring] resource attempt 12: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 12: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 12: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="11548") test_context_test.go:413: [TestMonitoring] resource attempt 12: patch completed in 20ms (resourceVersion="11548") test_context_test.go:367: [TestMonitoring] resource attempt 13: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 13: get completed in 20ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 13: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="12434") test_context_test.go:413: [TestMonitoring] resource attempt 13: patch completed in 22ms (resourceVersion="12434") e2e_test.go:38: installing Loki Operator from redhat-operators channel stable-6.6 test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-operators-redhat test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=namespaces "openshift-operators-redhat" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-operators-redhat test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 21ms (resourceVersion="14289") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 16ms (found=false, err=operatorgroups.operators.coreos.com "loki-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-operators-redhat/loki-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 27ms (resourceVersion="14313") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=subscriptions.operators.coreos.com "loki-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="14320") test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 19ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="14404") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 20ms (resourceVersion="14404") test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="14404") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 20ms (resourceVersion="14404") test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="14404") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 20ms (resourceVersion="14404") test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="14404") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 20ms (resourceVersion="14404") test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="14404") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 20ms (resourceVersion="14404") test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="15008") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 21ms (resourceVersion="15008") e2e_test.go:38: installing OpenShift Logging Operator from redhat-operators channel stable-6.6 test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-logging test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 18ms (found=false, err=namespaces "openshift-logging" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-logging test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 20ms (resourceVersion="15905") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 21ms (found=false, err=operatorgroups.operators.coreos.com "cluster-logging" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-logging/cluster-logging test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 21ms (resourceVersion="15930") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 18ms (found=false, err=subscriptions.operators.coreos.com "cluster-logging" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="15938") test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging (resourceVersion="16022") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 21ms (resourceVersion="16022") test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging (resourceVersion="16037") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 20ms (resourceVersion="16037") test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging (resourceVersion="16037") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 20ms (resourceVersion="16037") test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging (resourceVersion="16037") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 20ms (resourceVersion="16037") test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging (resourceVersion="16037") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 20ms (resourceVersion="16037") test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 21ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-logging/cluster-logging (resourceVersion="16103") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 21ms (resourceVersion="16103") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=namespaces "redhat-ods-monitoring" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="16676") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=configmaps "odh-observability-metrics-ca" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="16692") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 16ms (found=false, err=servicemonitors.monitoring.rhobs "odh-observability-metrics" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 20ms (resourceVersion="16701") e2e_test.go:38: created operator ServiceMonitor in namespace redhat-ods-monitoring for Service in odh-observability-system test_context_test.go:367: [TestMonitoring] resource attempt 1: getting services.platform.opendatahub.io/v1alpha1, Kind=Monitoring /default-monitoring test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 16ms (found=false, err=monitorings.services.platform.opendatahub.io "default-monitoring" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating services.platform.opendatahub.io/v1alpha1, Kind=Monitoring /default-monitoring test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 19ms (resourceVersion="16703") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=true, err=) helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Baseline_without_metrics helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_Traces_default_content === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_active_operand_namespace_and_readiness_contract monitoring_test.go:105: DSCI namespace handoff is not established in standalone module mode; Monitoring operand namespace is "redhat-ods-monitoring" === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Metrics_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Traces_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Alerting_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Perses_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/NodeMetrics_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/OpenTelemetry_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Usage_Logs_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Partial_signal_conditions_preserve_independent_traces helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Baseline_without_metrics/Target_Allocator_absent === RUN TestMonitoring/Baseline_without_metrics/Thanos_Querier_absent === RUN TestMonitoring/Baseline_without_metrics/Perses_absent === NAME TestMonitoring/Baseline_without_metrics helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_metrics_exporters_only monitoring_modes_test.go:35: Waiting for previous Prometheus pods to finish terminating helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_traces_exporters_only monitoring_modes_test.go:36: Waiting for previous Prometheus pods to finish terminating helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_combined_exporters_only_distinct_names monitoring_modes_test.go:37: Waiting for previous Prometheus pods to finish terminating helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_metrics_only_with_storage monitoring_modes_test.go:38: Waiting for previous Prometheus pods to finish terminating monitoring_modes_test.go:38: Waiting for Prometheus StatefulSet and pods to be ready helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_traces_only_with_storage monitoring_modes_test.go:39: Waiting for previous Prometheus pods to finish terminating helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_1_all_built-in_metrics_and_traces_with_storage monitoring_modes_test.go:40: Waiting for previous Prometheus pods to finish terminating monitoring_modes_test.go:40: Waiting for Prometheus StatefulSet and pods to be ready helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Metrics_ready helper_test.go:451: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:75: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:75: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Creation === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Configuration === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Prometheus_Self_ServiceMonitor_TLS_Fix helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_ownerReference_and_resourceVersion_stability === RUN TestMonitoring/Metrics_ready/Korrel8r === RUN TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources === RUN TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/reconciles_a_ready_TLS_query_service monitoring_korrel8r_test.go:46: same-namespace Korrel8r ingress probe denied: korrel8r-ingress-18e36e8856bf881b === RUN TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/serves_bounded_authenticated_API_requests monitoring_korrel8r_test.go:47: Korrel8r graph contract is not established: no ready collector workload pod is available as a known object === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Basic_Traces_Configuration helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Trace_Ingestion_always_uses_TLS_via_gateway helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Metrics_Exporters helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Traces_Exporters helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === NAME TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector monitoring_test.go:425: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_signal-specific_collector_contract === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_replicas === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OTLP_receiver_TLS_is_always_enabled === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Metrics_TLS_is_always_enabled_for_Prometheus_exporter === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Monitoring_TLS_is_always_enabled_for_internal_telemetry_exporter === RUN TestMonitoring/Metrics_ready/Target_Allocator === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_deployment_with_metrics helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_Service_and_ConfigMap helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_RBAC_configuration helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Thanos_Querier === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_deployment_with_metrics === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_authorization_boundary === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_namespace_isolation monitoring_test.go:1157: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:1157: thanos route evidence persona=anonymous http_status=401 prometheus_status="" series_count=0 namespaces=[] monitoring_test.go:1157: thanos route evidence persona=restricted http_status=403 prometheus_status="" series_count=0 namespaces=[] monitoring_test.go:1157: thanos route evidence persona=authorized-missing-namespace http_status=400 prometheus_status="" series_count=0 namespaces=[] monitoring_test.go:1157: Waiting for the authorized Thanos route to return namespace-scoped Prometheus data monitoring_test.go:1157: thanos route evidence persona=authorized http_status=200 prometheus_status="success" series_count=18 namespaces=[redhat-ods-monitoring] monitoring_test.go:1157: thanos route evidence persona=authorized-crafted-selector http_status=200 prometheus_status="success" series_count=18 namespaces=[redhat-ods-monitoring] monitoring_test.go:1157: thanos route evidence persona=authorized-post http_status=200 prometheus_status="success" series_count=18 namespaces=[redhat-ods-monitoring] monitoring_test.go:1157: thanos route evidence persona=authorized-dual-namespace-get http_status=403 prometheus_status="" series_count=0 namespaces=[] monitoring_test.go:1157: thanos route evidence persona=authorized-duplicate-namespace-post http_status=403 prometheus_status="" series_count=0 namespaces=[] === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_Prometheus_NetworkPolicy_allows_Thanos_Querier_on_gRPC_port === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_baseline_fixture_—_record_rollout_identity_and_health monitoring_test.go:1161: ✓ THANOS BASELINE RECORDED (independent of Perses) • Component: Thanos Querier • Rollout identity: resourceVersion=28219 generation=1 • CR status: ThanosQuerierAvailable=True • Query readiness: Deployment thanos-querier-data-science-thanos-querier ready replicas up • PromQL endpoint: Route data-science-thanos-querier-route → data-science-thanos-querier-proxy (reencrypt TLS) • PromQL response: query="up" status=success series=19 === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP monitoring_test.go:1162: LIMITATION (Component=Thanos): independent version-boundary testing is UNSUPPORTED and no version boundary is exercised here. COO packages Perses, Thanos and MonitoringStack under a single operator version, so the Thanos operand version cannot be pinned or swapped independently. Current rollout identity: resourceVersion=28219 generation=1. Reported as SKIP (not PASS) so this is not mistaken for boundary coverage; revisit once operand pinning is available. === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_component_health_check_with_failure-attribution_diagnostics monitoring_test.go:1163: ✓ Thanos health check passed; failure-attribution diagnostics wired: FAILURE ATTRIBUTION | Component=Thanos | Rollout=resourceVersion=28219 generation=1 | Assertion=Deployment thanos-querier-data-science-thanos-querier ready replicas | Diagnostics: oc -n redhat-ods-monitoring logs deployment/thanos-querier-data-science-thanos-querier ; oc -n redhat-ods-monitoring describe deployment thanos-querier-data-science-thanos-querier ; oc -n redhat-ods-monitoring get thanosquerier data-science-thanos-querier -o yaml === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Integration_—_Thanos_⇄_Perses_datasource_wiring monitoring_test.go:1166: ✓ THANOS ⇄ PERSES DATASOURCE WIRING OK • Dependent dashboards: PersesDatasource data-science-prometheus-datasource → HTTPProxy → https://data-science-thanos-querier-proxy.redhat-ods-monitoring.svc.cluster.local:8443 (namespace-enforcing) === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Integration_—_Perses_datasource_proxy_namespace_scoping monitoring_test.go:1167: perses→proxy authorized+namespace http=200 body="{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"up\",\"container\":\"alertmanager\",\"endpoint\":\"web\",\"instance\":\"10.132.0.55:9093\",\"job\":\"alertmanager…" monitoring_test.go:1167: perses→proxy authorized-but-foreign-namespace http=403 body="Forbidden (user=system:serviceaccount:redhat-ods-monitoring:perses-proxy-authorized-1793399019022240064, verb=get, resource=pods, subresource=)" monitoring_test.go:1167: perses→proxy multi-ns deny (allowed+foreign) http=403 body="Forbidden (user=system:serviceaccount:redhat-ods-monitoring:perses-proxy-authorized-1793399019022240064, verb=get, resource=pods, subresource=)" monitoring_test.go:1167: perses→proxy multi-ns allow http=200 body="{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"up\",\"container\":\"alertmanager\",\"endpoint\":\"web\",\"instance\":\"10.132.0.55:9093\",\"job\":\"alertmanager…" monitoring_test.go:1167: perses→proxy authorized-missing-namespace http=400 body="Bad Request. The request or configuration is malformed." monitoring_test.go:1167: perses→proxy no-token http=401 body="Unauthorized" monitoring_test.go:1167: perses→proxy denied-sa http=403 body="Forbidden (user=system:serviceaccount:redhat-ods-monitoring:perses-proxy-denied-1793399019022240064, verb=get, resource=pods, subresource=)" === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_restricted_resource_configuration monitoring_test.go:2731: MonitoringStack CR is Available — prerequisites met for prometheus-namespace-proxy === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_secure_proxy_authentication monitoring_test.go:2732: MonitoringStack CR is Available — prerequisites met for prometheus-namespace-proxy === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Monitoring_proxy_TLS_profile_rollout monitoring_test.go:2733: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction monitoring_test.go:2734: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-tenant-metrics" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 17ms (resourceVersion="29173") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 16ms (found=false, err=namespaces "tests-monitoring-tenant-metrics-foreign" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 16ms (resourceVersion="29189") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=deployments.apps "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 33ms (resourceVersion="29212") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=services "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 27ms (resourceVersion="29221") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=servicemonitors.monitoring.coreos.com "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 18ms (resourceVersion="29226") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=serviceaccounts "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 16ms (resourceVersion="29237") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=rolebindings.rbac.authorization.k8s.io "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 69ms (resourceVersion="29243") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=ServiceAccount redhat-ods-monitoring/tenant-metrics-foreign-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 18ms (found=false, err=serviceaccounts "tenant-metrics-foreign-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=ServiceAccount redhat-ods-monitoring/tenant-metrics-foreign-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 16ms (resourceVersion="29876") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics-foreign/tenant-metrics-foreign-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=rolebindings.rbac.authorization.k8s.io "tenant-metrics-foreign-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics-foreign/tenant-metrics-foreign-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 19ms (resourceVersion="29880") === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor monitoring_test.go:2735: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: getting /v1, Kind=ServiceAccount redhat-ods-monitoring/dcgm-https-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: get completed in 14ms (found=false, err=serviceaccounts "dcgm-https-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: creating /v1, Kind=ServiceAccount redhat-ods-monitoring/dcgm-https-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: create completed in 18ms (resourceVersion="30484") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-dcgm-https-18e36f12c627fde2/dcgm-https-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: get completed in 14ms (found=false, err=rolebindings.rbac.authorization.k8s.io "dcgm-https-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-dcgm-https-18e36f12c627fde2/dcgm-https-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor] resource attempt 1: create completed in 19ms (resourceVersion="30488") helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed monitoring_test.go:2736: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-tenant-metrics" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 16ms (resourceVersion="31658") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 18ms (found=false, err=namespaces "tests-monitoring-tenant-metrics-foreign" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 21ms (resourceVersion="31675") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 17ms (found=false, err=deployments.apps "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 34ms (resourceVersion="31697") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 16ms (found=false, err=services "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 26ms (resourceVersion="31707") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.coreos.com "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 21ms (resourceVersion="31712") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 14ms (found=false, err=serviceaccounts "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 18ms (resourceVersion="31720") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=rolebindings.rbac.authorization.k8s.io "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 61ms (resourceVersion="31753") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-otlp-identity test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-otlp-identity" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-otlp-identity test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 19ms (resourceVersion="31758") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=ServiceAccount redhat-ods-monitoring/otlp-identity-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 14ms (found=false, err=serviceaccounts "otlp-identity-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=ServiceAccount redhat-ods-monitoring/otlp-identity-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 18ms (resourceVersion="31781") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-otlp-identity/otlp-identity-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=rolebindings.rbac.authorization.k8s.io "otlp-identity-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-otlp-identity/otlp-identity-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 67ms (resourceVersion="31788") monitoring_test.go:2736: collector Prometheus scrape interval: 30s (Prometheus resource) monitoring_test.go:2736: collector warm-up pod=data-science-collector-collector-0 attributed=true pushStatus=200 pushFailed=false queryStatus=200 queryFailed=false seriesCount=5 monitoring_test.go:2736: collector warm-up pod=data-science-collector-collector-1 attributed=true pushStatus=200 pushFailed=false queryStatus=200 queryFailed=false seriesCount=8 monitoring_test.go:2736: observing OTLP delivery and tenant isolation for 1m40s (Prometheus scrape interval 30s) === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-0 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-1 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/OTLP_delivery === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/sender_attribution === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/forged_identity_sanitization === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/tenant_isolation === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_deployment === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_RBAC_configuration === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Namespace_isolation_POST_form_bypass monitoring_test.go:2739: POST-body namespace bypass is closed: the method gate rejects non-GET/HEAD (403) before prom-label-proxy === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization monitoring_test.go:2740: Operator metrics RBAC resources verified test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: getting /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: get completed in 16ms (found=true, err=) test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: getting /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patching /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca (resourceVersion="16702") test_context_test.go:413: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patch completed in 19ms (resourceVersion="33286") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patching monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics (resourceVersion="16701") test_context_test.go:413: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patch completed in 19ms (resourceVersion="16701") monitoring_test.go:2740: created operator ServiceMonitor in namespace redhat-ods-monitoring for Service in odh-observability-system === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthenticated_request_returns_401 monitoring_test.go:3362: operator metrics probe from redhat-ods-monitoring: status=401 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthorized_request_returns_403 monitoring_test.go:3372: operator metrics probe from redhat-ods-monitoring: status=403 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Authorized_request_returns_200 monitoring_test.go:3388: operator metrics probe from redhat-ods-monitoring: status=200 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-alertmanager-ingress === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-prometheus-cluster-proxy-ingress === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-thanos-querier-ingress === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_and_TargetAllocator_NetworkPolicy_contract monitoring_test.go:2742: no running instrumented KServe workload exporting OTLP to the collector; only selector probes will run monitoring_test.go:2742: OTLP selector probes use namespace redhat-ods-monitoring (enrolled=false) monitoring_test.go:2742: OTLP selector probes use namespace collector-network-policy-18e36fa13e562b69 (enrolled=true) === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_status === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_finalize === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/OpenShift_project === NAME TestMonitoring/Metrics_ready helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_PV_Backend === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_TempoMonolithic_CR_Creation_with_PV_backend === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_MLflow_integration_RBAC helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_Tempo_trace_export_RBAC helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Tempo_gateway_tenant_and_authentication_wiring helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === NAME TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_Cloud_Storage === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend test_context_test.go:367: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/s3-secret test_context_test.go:371: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: get completed in 14ms (found=false, err=secrets "s3-secret" not found) test_context_test.go:378: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/s3-secret test_context_test.go:387: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: create completed in 26ms (resourceVersion="37352") === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend test_context_test.go:367: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/gcs-secret test_context_test.go:371: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: get completed in 14ms (found=false, err=secrets "gcs-secret" not found) test_context_test.go:378: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/gcs-secret test_context_test.go:387: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: create completed in 17ms (resourceVersion="38745") 2026-10-30T22:47:15Z INFO override tempo configuration could potentially break the stack, use it carefully === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Instrumentation_CR_Traces_lifecycle === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Traces_Exporters_Reserved_Name_Validation === NAME TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Perses === RUN TestMonitoring/Perses/Group_8:_Perses monitoring_test.go:2094: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:2094: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_deployment_when_monitoring_is_managed === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_CR_configuration === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_NetworkPolicy_creation monitoring_thanos_network_policy_test.go:126: Perses: ready pod data-science-perses-0 has selector labels map[app.kubernetes.io/instance:data-science-perses app.kubernetes.io/managed-by:perses-operator app.kubernetes.io/name:data-science-perses] === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity monitoring_thanos_network_policy_test.go:126: querier: ready pod thanos-querier-data-science-thanos-querier-74b6964bcd-5zz58 has selector labels map[app.kubernetes.io/managed-by:observability-operator app.kubernetes.io/part-of:ThanosQuerier] === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/prometheus monitoring_thanos_network_policy_test.go:126: prometheus: ready pod prometheus-data-science-monitoringstack-0 has selector labels map[app.kubernetes.io/instance:data-science-monitoringstack app.kubernetes.io/name:prometheus] monitoring_thanos_network_policy_test.go:93: Thanos ingress probe prometheus from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/perses monitoring_thanos_network_policy_test.go:126: perses: ready pod data-science-perses-0 has selector labels map[app.kubernetes.io/name:data-science-perses instance:data-science-perses] monitoring_thanos_network_policy_test.go:93: Thanos ingress probe perses from redhat-ods-monitoring completed (allowed=false) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/korrel8r monitoring_thanos_network_policy_test.go:126: korrel8r: ready pod korrel8r-5cbcc89b8f-qjnqp has selector labels map[app.kubernetes.io/component:correlation app.kubernetes.io/name:korrel8r app.kubernetes.io/part-of:monitoring] monitoring_thanos_network_policy_test.go:93: Thanos ingress probe korrel8r from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/authenticated-proxy monitoring_thanos_network_policy_test.go:126: authenticated-proxy: ready pod data-science-thanos-querier-proxy-699986fdb7-vx7tr has selector labels map[app:data-science-thanos-querier-proxy] monitoring_thanos_network_policy_test.go:93: Thanos ingress probe authenticated-proxy from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/other-namespace-denied monitoring_thanos_network_policy_test.go:109: Thanos ingress probe untrusted from thanos-ingress-probe-18e3706f678e1b15 completed (allowed=false) === NAME TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity helper_test.go:392: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Creation_with_Traces === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Configuration === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_PersesDatasource_deployment_with_Prometheus === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_baseline_fixture_—_record_rollout_identity_and_health monitoring_test.go:2125: ✓ PERSES BASELINE RECORDED (independent of Thanos) • Component: Perses • Rollout identity: resourceVersion=43890 generation=2 • CR status: PersesAvailable=True • Dashboard config: .spec.config database+schemas present • Service ready: StatefulSet data-science-perses readyReplicas=1 • API: Pod data-science-perses-0 Running/Ready on :8080 === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP monitoring_test.go:2126: LIMITATION (Component=Perses): independent version-boundary testing is UNSUPPORTED and no version boundary is exercised here. COO packages Perses, Thanos and MonitoringStack under a single operator version, so the Perses operand version cannot be pinned or swapped independently. Current rollout identity: resourceVersion=43890 generation=2. Reported as SKIP (not PASS) so this is not mistaken for boundary coverage; revisit once operand pinning is available. === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_component_health_check_with_failure-attribution_diagnostics monitoring_test.go:2127: ✓ Perses health check passed; failure-attribution diagnostics wired: FAILURE ATTRIBUTION | Component=Perses | Rollout=resourceVersion=43890 generation=2 | Assertion=Pod data-science-perses-0 Running/Ready | Diagnostics: oc -n redhat-ods-monitoring logs data-science-perses-0 ; oc -n redhat-ods-monitoring describe pod data-science-perses-0 ; oc -n redhat-ods-monitoring get perses data-science-perses -o yaml === NAME TestMonitoring/Perses/Group_8:_Perses helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Webhooks === RUN TestMonitoring/Webhooks/Group_10:_Webhooks === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Setup monitoring_webhook_test.go:32: Setting up webhook tests: enabling monitoring and waiting for ready state monitoring_webhook_test.go:32: Webhook tests setup complete: monitoring is enabled and ready === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_namespace === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors monitoring_webhook_test.go:34: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:34: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: create completed in 17ms (resourceVersion="45447") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: get completed in 17ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: create completed in 19ms (resourceVersion="45463") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: get completed in 14ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: create completed in 18ms (resourceVersion="45472") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels monitoring_webhook_test.go:35: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:35: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: create completed in 17ms (resourceVersion="45523") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: get completed in 17ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: create completed in 252ms (resourceVersion="45551") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: create completed in 132ms (resourceVersion="45555") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection monitoring_webhook_test.go:36: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:36: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: create completed in 17ms (resourceVersion="45595") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: get completed in 17ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: create completed in 132ms (resourceVersion="45619") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: get completed in 14ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: create completed in 137ms (resourceVersion="45622") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update monitoring_webhook_test.go:37: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:37: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: create completed in 17ms (resourceVersion="45672") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: get completed in 14ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: create completed in 21ms (resourceVersion="45688") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: get completed in 16ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: create completed in 21ms (resourceVersion="45695") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels monitoring_webhook_test.go:38: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:38: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: create completed in 23ms (resourceVersion="45741") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: get completed in 20ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: create completed in 24ms (resourceVersion="45754") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: get completed in 14ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: create completed in 24ms (resourceVersion="45762") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace monitoring_webhook_test.go:39: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:39: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: create completed in 19ms (resourceVersion="45807") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: create completed in 43ms (resourceVersion="45823") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: get completed in 25ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: create completed in 22ms (resourceVersion="45830") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace monitoring_webhook_test.go:40: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:40: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: create completed in 17ms (resourceVersion="45868") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: get completed in 14ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: create completed in 24ms (resourceVersion="45884") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: get completed in 25ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: create completed in 27ms (resourceVersion="45892") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out monitoring_webhook_test.go:41: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:41: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: create completed in 17ms (resourceVersion="45924") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: get completed in 14ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: create completed in 20ms (resourceVersion="45940") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: get completed in 18ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: create completed in 19ms (resourceVersion="45948") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency monitoring_webhook_test.go:42: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:42: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: create completed in 18ms (resourceVersion="46001") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: get completed in 17ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: create completed in 25ms (resourceVersion="46018") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: create completed in 22ms (resourceVersion="46023") === RUN TestMonitoring/Usage_Logs_Collection === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_Collector_not_deployed_without_usage_logs_config helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_UsageLogs_resources test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_UsageLogs_resources] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-usage-logs-loki-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_UsageLogs_resources] resource attempt 1: get completed in 14ms (found=false, err=secrets "test-usage-logs-loki-secret" not found) test_context_test.go:378: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_UsageLogs_resources] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/test-usage-logs-loki-secret test_context_test.go:387: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_UsageLogs_resources] resource attempt 1: create completed in 22ms (resourceVersion="46538") === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_usage_logs_LokiStack_deployment === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_usage_logs_LokiStack_configuration === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_deployment === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_configuration === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_RBAC === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_network_policies === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_log_forwarding_resources test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_log_forwarding_resources] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-logs-loki-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_log_forwarding_resources] resource attempt 1: get completed in 15ms (found=false, err=secrets "test-logs-loki-secret" not found) test_context_test.go:378: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_log_forwarding_resources] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/test-logs-loki-secret test_context_test.go:387: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_log_forwarding_resources] resource attempt 1: create completed in 17ms (resourceVersion="47470") === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_log_forwarding_LokiStack_deployment === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_log_forwarding_LokiStack_configuration === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-logs-loki-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract] resource attempt 1: get completed in 15ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract] resource attempt 1: patching /v1, Kind=Secret redhat-ods-monitoring/test-logs-loki-secret (resourceVersion="47470") test_context_test.go:413: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract] resource attempt 1: patch completed in 17ms (resourceVersion="47470") monitoring_usage_logs_test.go:104: ClusterLogForwarder evidence: name=data-science-cluster-log-forwarder vector_ready=3 vector_desired=3 === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_log_forwarding_LokiStack_removal === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_Collector_serving_certificate_rotation === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector) test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: get completed in 15ms (found=false, err=secrets "test-loki-lifecycle-secret" not found) test_context_test.go:378: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret test_context_test.go:387: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: create completed in 17ms (resourceVersion="50108") test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: get completed in 14ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: patching /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret (resourceVersion="50108") test_context_test.go:413: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: patch completed in 16ms (resourceVersion="50108") helper_test.go:451: Waiting for previous Prometheus pods to finish terminating === NAME TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection monitoring_phase_test.go:39: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Metrics_replicas monitoring_test.go:3637: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:3637: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Target_Allocator monitoring_test.go:3642: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3643: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses monitoring_test.go:3647: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3648: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses_datasource monitoring_test.go:3652: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3653: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Prometheus_rules monitoring_test.go:3657: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled monitoring_test.go:3662: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:3662: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3663: Disabling monitoring to test webhook behavior when monitoring is disabled monitoring_test.go:3663: Monitoring disabled, proceeding to test webhook behavior monitoring_test.go:3663: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_test.go:3663: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: get completed in 14ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: create completed in 17ms (resourceVersion="61763") test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: get completed in 14ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:381: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: create failed after 5.017s: Internal error occurred: failed calling webhook "podmonitor-injector.opendatahub.io": failed to call webhook: Post "https://odh-observability-webhook.odh-observability-system.svc:9443/mutate-prometheus-monitors?timeout=5s": context deadline exceeded test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 2: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 2: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 2: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:381: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 2: create failed after 5.015s: Internal error occurred: failed calling webhook "podmonitor-injector.opendatahub.io": failed to call webhook: Post "https://odh-observability-webhook.odh-observability-system.svc:9443/mutate-prometheus-monitors?timeout=5s": context deadline exceeded test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 3: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 3: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 3: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:381: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 3: create failed after 5.015s: Internal error occurred: failed calling webhook "podmonitor-injector.opendatahub.io": failed to call webhook: Post "https://odh-observability-webhook.odh-observability-system.svc:9443/mutate-prometheus-monitors?timeout=5s": context deadline exceeded test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 4: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 4: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 4: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 4: create completed in 243ms (resourceVersion="61872") test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: get completed in 14ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: create completed in 133ms (resourceVersion="61874") monitoring_test.go:3663: Webhook correctly skipped injection when monitoring was disabled === RUN TestMonitoring/Disabled === RUN TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled === RUN TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled/Validate_monitoring_service_disabled --- PASS: TestMonitoring (4431.84s) --- PASS: TestMonitoring/openshift-cert-manager-operator (0.15s) --- PASS: TestMonitoring/install-dependent-operators (0.00s) --- PASS: TestMonitoring/install-dependent-operators/opentelemetry-product (44.72s) --- PASS: TestMonitoring/install-dependent-operators/tempo-product (48.80s) --- PASS: TestMonitoring/install-dependent-operators/cluster-observability-operator (54.91s) --- PASS: TestMonitoring/Baseline_without_metrics (43.17s) --- PASS: TestMonitoring/Baseline_without_metrics/Base_Configuration (0.23s) --- PASS: TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration (0.23s) --- PASS: TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_Traces_default_content (0.11s) --- SKIP: TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_active_operand_namespace_and_readiness_contract (0.04s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions (42.01s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions (42.01s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Metrics_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Traces_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Alerting_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Perses_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/NodeMetrics_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/OpenTelemetry_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Usage_Logs_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Partial_signal_conditions_preserve_independent_traces (41.22s) --- PASS: TestMonitoring/Baseline_without_metrics/Target_Allocator_absent (0.18s) --- PASS: TestMonitoring/Baseline_without_metrics/Thanos_Querier_absent (0.19s) --- PASS: TestMonitoring/Baseline_without_metrics/Perses_absent (0.16s) --- PASS: TestMonitoring/Observability_Modes (343.77s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes (343.77s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_metrics_exporters_only (39.41s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_traces_exporters_only (31.31s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_combined_exporters_only_distinct_names (37.34s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_metrics_only_with_storage (73.09s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_traces_only_with_storage (72.03s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_1_all_built-in_metrics_and_traces_with_storage (90.39s) --- PASS: TestMonitoring/Metrics_ready (1627.07s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack (91.83s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack (91.83s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Creation (20.61s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Configuration (0.04s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Prometheus_Self_ServiceMonitor_TLS_Fix (0.42s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_ownerReference_and_resourceVersion_stability (70.76s) --- PASS: TestMonitoring/Metrics_ready/Korrel8r (10.59s) --- PASS: TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources (10.59s) --- PASS: TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/reconciles_a_ready_TLS_query_service (10.42s) --- SKIP: TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/serves_bounded_authenticated_API_requests (0.17s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector (364.19s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector (364.19s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations (262.87s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Basic_Traces_Configuration (57.57s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Trace_Ingestion_always_uses_TLS_via_gateway (69.81s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Metrics_Exporters (65.71s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Traces_Exporters (69.78s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_signal-specific_collector_contract (0.15s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_replicas (67.56s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OTLP_receiver_TLS_is_always_enabled (0.28s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Metrics_TLS_is_always_enabled_for_Prometheus_exporter (0.24s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Monitoring_TLS_is_always_enabled_for_internal_telemetry_exporter (0.24s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator (31.69s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator (31.69s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics (31.69s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_deployment_with_metrics (31.01s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_Service_and_ConfigMap (0.27s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_RBAC_configuration (0.41s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier (16.47s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier (16.47s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_deployment_with_metrics (0.18s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_authorization_boundary (0.31s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_namespace_isolation (2.66s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_Prometheus_NetworkPolicy_allows_Thanos_Querier_on_gRPC_port (0.14s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos (0.77s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_baseline_fixture_—_record_rollout_identity_and_health (0.63s) --- SKIP: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP (0.02s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_component_health_check_with_failure-attribution_diagnostics (0.12s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Integration_—_Thanos_⇄_Perses_datasource_wiring (0.10s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Integration_—_Perses_datasource_proxy_namespace_scoping (12.31s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC (1040.54s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC (1040.54s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_restricted_resource_configuration (0.33s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_secure_proxy_authentication (0.39s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Monitoring_proxy_TLS_profile_rollout (68.86s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction (102.11s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Built-in_DCGM_opt-out_with_verified_HTTPS_monitor (205.63s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed (310.33s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-0 (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-1 (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/OTLP_delivery (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/sender_attribution (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/forged_identity_sanitization (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/tenant_isolation (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_deployment (24.75s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_RBAC_configuration (0.28s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Namespace_isolation_POST_form_bypass (0.64s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization (7.13s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthenticated_request_returns_401 (0.50s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthorized_request_returns_403 (0.64s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Authorized_request_returns_200 (0.61s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle (38.71s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-alertmanager-ingress (14.22s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-prometheus-cluster-proxy-ingress (12.21s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-thanos-querier-ingress (12.21s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_and_TargetAllocator_NetworkPolicy_contract (257.22s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization (24.17s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace (0.28s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_status (0.17s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_finalize (0.15s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/OpenShift_project (0.23s) --- PASS: TestMonitoring/Traces_with_PV_Backend (166.03s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend (166.03s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_TempoMonolithic_CR_Creation_with_PV_backend (58.92s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_MLflow_integration_RBAC (6.51s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_Tempo_trace_export_RBAC (35.07s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Tempo_gateway_tenant_and_authentication_wiring (65.34s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage (346.19s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage (346.19s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage (309.10s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend (156.59s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend (152.51s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Instrumentation_CR_Traces_lifecycle (26.62s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Traces_Exporters_Reserved_Name_Validation (4.11s) --- PASS: TestMonitoring/Perses (270.68s) --- PASS: TestMonitoring/Perses/Group_8:_Perses (270.68s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle (26.94s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_deployment_when_monitoring_is_managed (26.64s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_CR_configuration (0.19s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_NetworkPolicy_creation (0.11s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity (27.35s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/prometheus (4.18s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/perses (8.20s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/korrel8r (4.17s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/authenticated-proxy (2.17s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/other-namespace-denied (8.36s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces (137.26s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Creation_with_Traces (67.53s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Configuration (69.51s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_PersesDatasource_deployment_with_Prometheus (0.22s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses (0.35s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_baseline_fixture_—_record_rollout_identity_and_health (0.22s) --- SKIP: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP (0.02s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_component_health_check_with_failure-attribution_diagnostics (0.12s) --- PASS: TestMonitoring/Webhooks (107.81s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks (107.81s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Setup (51.06s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_namespace (0.04s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors (6.39s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels (6.72s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection (6.61s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update (4.69s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels (6.80s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace (6.39s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace (6.39s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out (6.35s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency (6.36s) --- PASS: TestMonitoring/Usage_Logs_Collection (708.03s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection (708.03s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_Collector_not_deployed_without_usage_logs_config (14.65s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_UsageLogs_resources (120.44s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_usage_logs_LokiStack_deployment (0.07s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_usage_logs_LokiStack_configuration (0.03s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_deployment (0.21s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_configuration (0.04s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_RBAC (0.09s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_network_policies (0.06s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_log_forwarding_resources (103.02s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_log_forwarding_LokiStack_deployment (0.16s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_log_forwarding_LokiStack_configuration (0.03s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract (10.63s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_log_forwarding_LokiStack_removal (0.21s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_Collector_serving_certificate_rotation (302.24s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector) (110.93s) --- PASS: TestMonitoring/Lifecycle (688.08s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle (688.08s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Metrics_replicas (80.71s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Target_Allocator (139.74s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses (126.86s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses_datasource (147.40s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Prometheus_rules (92.12s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled (101.25s) --- PASS: TestMonitoring/Disabled (0.41s) --- PASS: TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled (0.41s) --- PASS: TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled/Validate_monitoring_service_disabled (0.41s) PASS ok github.com/opendatahub-io/odh-observability/tests/e2e 4431.852s