go test ./tests/e2e/ -v -timeout 120m -count=1 -run ^TestMonitoring$ -timeout 90m -api-mode module -monitoring-cr-name default-monitoring -install-operators=true -olm-timeout 15m -eventually-timeout 10m go: downloading github.com/onsi/gomega v1.42.0 go: downloading github.com/opendatahub-io/odh-platform-utilities v0.5.0 go: downloading github.com/openshift/api v0.0.0-20261002115817-f8795cdde518 go: downloading github.com/openshift/library-go v0.0.0-20261001144932-192662696fc7 go: downloading github.com/stretchr/testify v1.11.1 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading k8s.io/api v0.37.1 go: downloading k8s.io/apimachinery v0.37.1 go: downloading k8s.io/client-go v0.37.1 go: downloading sigs.k8s.io/controller-runtime v0.25.1 go: downloading github.com/itchyny/gojq v0.12.19 go: downloading github.com/google/go-cmp v0.7.0 go: downloading k8s.io/apiserver v0.37.1 go: downloading k8s.io/klog/v2 v2.140.0 go: downloading sigs.k8s.io/randfill v1.0.0 go: downloading k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 go: downloading sigs.k8s.io/yaml v1.6.0 go: downloading k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad go: downloading sigs.k8s.io/structured-merge-diff/v6 v6.4.2 go: downloading github.com/evanphx/json-patch/v5 v5.9.11 go: downloading github.com/go-logr/logr v1.4.3 go: downloading gopkg.in/evanphx/json-patch.v4 v4.13.0 go: downloading github.com/go-logr/zapr v1.3.0 go: downloading go.uber.org/zap v1.27.1 go: downloading go.yaml.in/yaml/v3 v3.0.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading golang.org/x/net v0.58.0 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/itchyny/timefmt-go v0.1.8 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 go: downloading k8s.io/streaming v0.37.1 go: downloading go.yaml.in/yaml/v2 v2.4.4 go: downloading github.com/spf13/pflag v1.0.10 go: downloading golang.org/x/term v0.46.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/k8s-manifest-kit/engine v0.2.1-0.20260302092700-39c16f95d249 go: downloading github.com/fxamacker/cbor/v2 v2.9.1 go: downloading github.com/google/gnostic-models v0.7.1 go: downloading google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af go: downloading golang.org/x/time v0.15.0 go: downloading golang.org/x/oauth2 v0.36.0 go: downloading github.com/go-openapi/jsonreference v1.0.0 go: downloading github.com/go-openapi/swag v0.27.1 go: downloading golang.org/x/sys v0.48.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee go: downloading github.com/x448/float16 v0.8.4 go: downloading github.com/k8s-manifest-kit/pkg v0.2.1-0.20260302085906-6946a2e8e1d5 go: downloading github.com/go-openapi/jsonpointer v1.0.0 go: downloading github.com/go-openapi/swag/cmdutils v0.27.1 go: downloading github.com/go-openapi/swag/conv v0.27.1 go: downloading github.com/go-openapi/swag/fileutils v0.27.1 go: downloading github.com/go-openapi/swag/jsonutils v0.27.1 go: downloading github.com/go-openapi/swag/loading v0.27.1 go: downloading github.com/go-openapi/swag/mangling v0.27.1 go: downloading github.com/go-openapi/swag/netutils v0.27.1 go: downloading github.com/go-openapi/swag/stringutils v0.27.1 go: downloading github.com/go-openapi/swag/typeutils v0.27.1 go: downloading github.com/go-openapi/swag/yamlutils v0.27.1 go: downloading golang.org/x/text v0.42.0 go: downloading github.com/prometheus/client_golang v1.24.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/moby/spdystream v0.5.1 go: downloading github.com/emicklei/go-restful/v3 v3.13.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.2 go: downloading github.com/prometheus/common v0.70.0 go: downloading github.com/prometheus/procfs v0.21.1 go: downloading github.com/go-openapi/swag/pools v0.27.1 === RUN TestMonitoring e2e_test.go:32: detected 3 schedulable nodes — expecting 2 replicas === RUN TestMonitoring/openshift-cert-manager-operator === NAME TestMonitoring test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /cert-manager-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=true, err=) test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription cert-manager-operator/openshift-cert-manager-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 1: patching operators.coreos.com/v1alpha1, Kind=Subscription cert-manager-operator/openshift-cert-manager-operator (resourceVersion="15983") test_context_test.go:413: [TestMonitoring] resource attempt 1: patch completed in 19ms (resourceVersion="15983") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=ClusterRole /odh-observability-opentelemetry-cert-manager test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=false, err=clusterroles.rbac.authorization.k8s.io "odh-observability-opentelemetry-cert-manager" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=ClusterRole /odh-observability-opentelemetry-cert-manager test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 17ms (resourceVersion="16927") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding /odh-observability-opentelemetry-cert-manager test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 14ms (found=false, err=clusterrolebindings.rbac.authorization.k8s.io "odh-observability-opentelemetry-cert-manager" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding /odh-observability-opentelemetry-cert-manager test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 17ms (resourceVersion="16928") === RUN TestMonitoring/install-dependent-operators === RUN TestMonitoring/install-dependent-operators/cluster-observability-operator === PAUSE TestMonitoring/install-dependent-operators/cluster-observability-operator === RUN TestMonitoring/install-dependent-operators/tempo-product === PAUSE TestMonitoring/install-dependent-operators/tempo-product === RUN TestMonitoring/install-dependent-operators/opentelemetry-product === PAUSE TestMonitoring/install-dependent-operators/opentelemetry-product === CONT TestMonitoring/install-dependent-operators/cluster-observability-operator === NAME TestMonitoring test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-cluster-observability-operator === CONT TestMonitoring/install-dependent-operators/opentelemetry-product === CONT TestMonitoring/install-dependent-operators/tempo-product === NAME TestMonitoring test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-tempo-operator test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-opentelemetry-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=false, err=namespaces "openshift-opentelemetry-operator" not found) test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=false, err=namespaces "openshift-tempo-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-tempo-operator test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-opentelemetry-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=false, err=namespaces "openshift-cluster-observability-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-cluster-observability-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 17ms (resourceVersion="16929") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 22ms (resourceVersion="16930") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 22ms (resourceVersion="16931") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 23ms (found=false, err=operatorgroups.operators.coreos.com "cluster-observability-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 23ms (found=false, err=operatorgroups.operators.coreos.com "opentelemetry-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 24ms (found=false, err=operatorgroups.operators.coreos.com "tempo-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-tempo-operator/tempo-product test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="16985") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 23ms (resourceVersion="16986") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 21ms (resourceVersion="16987") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 19ms (found=false, err=subscriptions.operators.coreos.com "opentelemetry-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 18ms (found=false, err=subscriptions.operators.coreos.com "cluster-observability-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 20ms (found=false, err=subscriptions.operators.coreos.com "tempo-product" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="16999") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 17ms (resourceVersion="17000") test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="17001") test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="17078") test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 15ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="17088") test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 15ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17050") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 17ms (resourceVersion="17078") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 18ms (resourceVersion="17088") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 18ms (resourceVersion="17050") test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="17097") test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="17107") test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 25ms (resourceVersion="17097") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 23ms (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 25ms (resourceVersion="17107") test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 22ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="17097") test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 22ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17131") test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 23ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="17107") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 19ms (resourceVersion="17097") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 21ms (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 20ms (resourceVersion="17107") test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="17097") test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17131") test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-opentelemetry-operator/opentelemetry-product (resourceVersion="17193") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 21ms (resourceVersion="17097") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 18ms (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 19ms (resourceVersion="17193") test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 25ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="17097") test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 25ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 22ms (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 22ms (resourceVersion="17097") test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator test_context_test.go:367: [TestMonitoring] resource attempt 7: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 26ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17131") test_context_test.go:371: [TestMonitoring] resource attempt 7: get completed in 29ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 7: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-cluster-observability-operator/cluster-observability-operator (resourceVersion="17398") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 20ms (resourceVersion="17398") test_context_test.go:413: [TestMonitoring] resource attempt 7: patch completed in 23ms (resourceVersion="17131") test_context_test.go:367: [TestMonitoring] resource attempt 8: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 8: get completed in 24ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 8: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17131") test_context_test.go:413: [TestMonitoring] resource attempt 8: patch completed in 22ms (resourceVersion="17131") test_context_test.go:367: [TestMonitoring] resource attempt 9: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product test_context_test.go:371: [TestMonitoring] resource attempt 9: get completed in 27ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 9: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-tempo-operator/tempo-product (resourceVersion="17654") test_context_test.go:413: [TestMonitoring] resource attempt 9: patch completed in 30ms (resourceVersion="17654") e2e_test.go:32: installing Loki Operator from redhat-operators channel stable-6.6 test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /openshift-operators-redhat test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 17ms (found=false, err=namespaces "openshift-operators-redhat" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /openshift-operators-redhat test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="20057") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1, Kind=OperatorGroup openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 20ms (found=false, err=operatorgroups.operators.coreos.com "loki-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1, Kind=OperatorGroup openshift-operators-redhat/loki-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="20080") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 21ms (found=false, err=subscriptions.operators.coreos.com "loki-operator" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 31ms (resourceVersion="20087") test_context_test.go:367: [TestMonitoring] resource attempt 2: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 2: get completed in 17ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 2: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="20209") test_context_test.go:413: [TestMonitoring] resource attempt 2: patch completed in 25ms (resourceVersion="20209") test_context_test.go:367: [TestMonitoring] resource attempt 3: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 3: get completed in 23ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 3: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="20319") test_context_test.go:413: [TestMonitoring] resource attempt 3: patch completed in 25ms (resourceVersion="20319") test_context_test.go:367: [TestMonitoring] resource attempt 4: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 4: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 4: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="20350") test_context_test.go:413: [TestMonitoring] resource attempt 4: patch completed in 24ms (resourceVersion="20350") test_context_test.go:367: [TestMonitoring] resource attempt 5: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 5: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 5: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="20350") test_context_test.go:413: [TestMonitoring] resource attempt 5: patch completed in 19ms (resourceVersion="20350") test_context_test.go:367: [TestMonitoring] resource attempt 6: getting operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator test_context_test.go:371: [TestMonitoring] resource attempt 6: get completed in 18ms (found=true, err=) test_context_test.go:404: [TestMonitoring] resource attempt 6: patching operators.coreos.com/v1alpha1, Kind=Subscription openshift-operators-redhat/loki-operator (resourceVersion="20491") test_context_test.go:413: [TestMonitoring] resource attempt 6: patch completed in 19ms (resourceVersion="20491") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=false, err=namespaces "redhat-ods-monitoring" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 25ms (resourceVersion="21229") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 19ms (found=false, err=configmaps "odh-observability-metrics-ca" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 23ms (resourceVersion="21243") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 20ms (found=false, err=servicemonitors.monitoring.rhobs "odh-observability-metrics" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 18ms (resourceVersion="21251") e2e_test.go:32: created operator ServiceMonitor in namespace redhat-ods-monitoring for Service in odh-observability-system test_context_test.go:367: [TestMonitoring] resource attempt 1: getting services.platform.opendatahub.io/v1alpha1, Kind=Monitoring /default-monitoring test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 23ms (found=false, err=monitorings.services.platform.opendatahub.io "default-monitoring" not found) test_context_test.go:378: [TestMonitoring] resource attempt 1: creating services.platform.opendatahub.io/v1alpha1, Kind=Monitoring /default-monitoring test_context_test.go:387: [TestMonitoring] resource attempt 1: create completed in 20ms (resourceVersion="21256") test_context_test.go:367: [TestMonitoring] resource attempt 1: getting /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:371: [TestMonitoring] resource attempt 1: get completed in 15ms (found=true, err=) helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Baseline_without_metrics helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_Traces_default_content === RUN TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_active_operand_namespace_and_readiness_contract monitoring_test.go:105: DSCI namespace handoff is not established in standalone module mode; Monitoring operand namespace is "redhat-ods-monitoring" === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Metrics_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Traces_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Alerting_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Perses_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/NodeMetrics_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/OpenTelemetry_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Usage_Logs_negative_conditions === RUN TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Partial_signal_conditions_preserve_independent_traces helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Baseline_without_metrics/Target_Allocator_absent === RUN TestMonitoring/Baseline_without_metrics/Thanos_Querier_absent === RUN TestMonitoring/Baseline_without_metrics/Perses_absent === NAME TestMonitoring/Baseline_without_metrics helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_metrics_exporters_only monitoring_modes_test.go:35: Waiting for previous Prometheus pods to finish terminating test_context_test.go:166: Timed out after 600.001s. Mode 2 metrics: metrics pipeline exporters must be exactly [otlphttp/metrics-external] Expected : &{map[apiVersion:opentelemetry.io/v1beta1 kind:OpenTelemetryCollector metadata:map[annotations:map[platform.opendatahub.io/instance.generation:4 platform.opendatahub.io/instance.name:default-monitoring platform.opendatahub.io/instance.uid:f0164913-977c-4091-ab5c-df2c37ecec26 platform.opendatahub.io/type:OpenDataHub platform.opendatahub.io/version:0.1.0] creationTimestamp:2026-10-30T02:57:46Z generation:2 labels:map[platform.opendatahub.io/part-of:monitoring] managedFields:[map[apiVersion:opentelemetry.io/v1beta1 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[f:platform.opendatahub.io/instance.generation:map[] f:platform.opendatahub.io/instance.name:map[] f:platform.opendatahub.io/instance.uid:map[] f:platform.opendatahub.io/type:map[] f:platform.opendatahub.io/version:map[]] f:labels:map[f:platform.opendatahub.io/part-of:map[]] f:ownerReferences:map[k:{"uid":"f0164913-977c-4091-ab5c-df2c37ecec26"}:map[]]] f:spec:map[f:config:map[f:exporters:map[f:otlphttp/metrics-external:map[.:map[] f:endpoint:map[] f:tls:map[.:map[] f:insecure:map[]]] f:prometheus/monitoring:map[.:map[] f:endpoint:map[] f:tls:map[.:map[] f:cert_file:map[] f:key_file:map[]]]] f:extensions:map[f:bearertokenauth:map[.:map[] f:filename:map[]]] f:processors:map[f:batch:map[.:map[] f:send_batch_size:map[]] f:k8sattributes:map[] f:k8sattributes/otlp:map[.:map[] f:pod_association:map[]] f:memory_limiter:map[.:map[] f:check_interval:map[] f:limit_mib:map[] f:spike_limit_mib:map[]] f:resourcedetection:map[.:map[] f:detectors:map[]] f:transform/exclude_dcgm_k8s_labels:map[.:map[] f:error_mode:map[] f:metric_statements:map[]] f:transform/gpu_metrics:map[.:map[] f:error_mode:map[] f:metric_statements:map[]] f:transform/sanitize_k8s_labels:map[.:map[] f:error_mode:map[] f:metric_statements:map[]] f:transform/sanitize_otlp_identity:map[.:map[] f:error_mode:map[] f:metric_statements:map[]]] f:receivers:map[f:otlp:map[.:map[] f:protocols:map[.:map[] f:grpc:map[] f:http:map[]]] f:prometheus:map[.:map[] f:config:map[.:map[] f:scrape_configs:map[]]] f:prometheus/self:map[.:map[] f:config:map[.:map[] f:scrape_configs:map[]]]] f:service:map[f:extensions:map[] f:pipelines:map[f:metrics/internal:map[.:map[] f:exporters:map[] f:receivers:map[]] f:metrics/otlp:map[.:map[] f:exporters:map[] f:processors:map[] f:receivers:map[]] f:metrics/scrape:map[.:map[] f:exporters:map[] f:processors:map[] f:receivers:map[]]] f:telemetry:map[f:metrics:map[.:map[] f:readers:map[]]]]] f:mode:map[] f:networkPolicy:map[f:enabled:map[]] f:replicas:map[] f:resources:map[f:limits:map[f:cpu:map[] f:memory:map[]] f:requests:map[f:cpu:map[] f:memory:map[]]] f:targetAllocator:map[f:allowInsecureAuthSecrets:map[] f:enabled:map[] f:mtls:map[f:enabled:map[] f:useCertManager:map[]] f:prometheusCR:map[f:denyFSAccessThroughSMs:map[] f:enabled:map[] f:podMonitorSelector:map[] f:secretNamespaces:map[] f:serviceMonitorSelector:map[]] f:serviceAccount:map[]] f:volumeMounts:map[] f:volumes:map[]]] manager:monitoring operation:Apply time:2026-10-30T02:57:48Z] map[apiVersion:opentelemetry.io/v1beta1 fieldsType:FieldsV1 fieldsV1:map[f:status:map[.:map[] f:conditions:map[] f:image:map[] f:observedGeneration:map[] f:scale:map[.:map[] f:replicas:map[] f:selector:map[] f:statusReplicas:map[]] f:version:map[]]] manager:opentelemetry-operator operation:Update subresource:status time:2026-10-30T02:57:50Z]] name:data-science-collector namespace:redhat-ods-monitoring ownerReferences:[map[apiVersion:services.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:true controller:true kind:Monitoring name:default-monitoring uid:f0164913-977c-4091-ab5c-df2c37ecec26]] resourceVersion:22198 uid:77cd52fd-aa95-4d4a-a4dc-cd5954774266] spec:map[config:map[exporters:map[otlphttp/metrics-external:map[endpoint:http://metrics-external.example.svc:4318 tls:map[insecure:true]] prometheus/monitoring:map[endpoint:0.0.0.0:8890 tls:map[cert_file:/etc/otel-collector/monitoring-tls/tls.crt key_file:/etc/otel-collector/monitoring-tls/tls.... Gomega truncated this representation as it exceeds 'format.MaxLength'. Consider having the object provide a custom 'GomegaStringer' representation or adjust the parameters in Gomega's 'format' package. Learn more here: https://onsi.github.io/gomega/#adjusting-output to match expression : (.spec.config.exporters | has("otlphttp/metrics-external")) and (.spec.config.exporters | has("prometheus") | not) and (.spec.config.service.pipelines.metrics.exporters | length == 1 and contains(["otlphttp/metrics-external"])) helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_traces_exporters_only monitoring_modes_test.go:36: Waiting for previous Prometheus pods to finish terminating helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_combined_exporters_only_distinct_names monitoring_modes_test.go:37: Waiting for previous Prometheus pods to finish terminating test_context_test.go:166: Timed out after 600.001s. Mode 2 combined: metrics/traces pipelines must be exactly the distinct custom exporters Expected : &{map[apiVersion:opentelemetry.io/v1beta1 kind:OpenTelemetryCollector metadata:map[annotations:map[platform.opendatahub.io/instance.generation:8 platform.opendatahub.io/instance.name:default-monitoring platform.opendatahub.io/instance.uid:f0164913-977c-4091-ab5c-df2c37ecec26 platform.opendatahub.io/type:OpenDataHub platform.opendatahub.io/version:0.1.0] creationTimestamp:2026-10-30T03:08:26Z generation:2 labels:map[platform.opendatahub.io/part-of:monitoring] managedFields:[map[apiVersion:opentelemetry.io/v1beta1 fieldsType:FieldsV1 fieldsV1:map[f:metadata:map[f:annotations:map[f:platform.opendatahub.io/instance.generation:map[] f:platform.opendatahub.io/instance.name:map[] f:platform.opendatahub.io/instance.uid:map[] f:platform.opendatahub.io/type:map[] f:platform.opendatahub.io/version:map[]] f:labels:map[f:platform.opendatahub.io/part-of:map[]] f:ownerReferences:map[k:{"uid":"f0164913-977c-4091-ab5c-df2c37ecec26"}:map[]]] f:spec:map[f:config:map[f:exporters:map[f:otlphttp/metrics-external:map[.:map[] f:endpoint:map[] f:tls:map[.:map[] f:insecure:map[]]] f:otlphttp/traces-external:map[.:map[] f:endpoint:map[] f:tls:map[.:map[] f:insecure:map[]]] f:prometheus/monitoring:map[.:map[] f:endpoint:map[] f:tls:map[.:map[] f:cert_file:map[] f:key_file:map[]]]] f:extensions:map[f:bearertokenauth:map[.:map[] f:filename:map[]]] f:processors:map[f:batch:map[.:map[] f:send_batch_size:map[]] f:k8sattributes:map[] f:k8sattributes/otlp:map[.:map[] f:pod_association:map[]] f:memory_limiter:map[.:map[] f:check_interval:map[] f:limit_mib:map[] f:spike_limit_mib:map[]] f:resourcedetection:map[.:map[] f:detectors:map[]] f:transform/exclude_dcgm_k8s_labels:map[.:map[] f:error_mode:map[] f:metric_statements:map[]] f:transform/gpu_metrics:map[.:map[] f:error_mode:map[] f:metric_statements:map[]] f:transform/sanitize_k8s_labels:map[.:map[] f:error_mode:map[] f:metric_statements:map[]] f:transform/sanitize_otlp_identity:map[.:map[] f:error_mode:map[] f:metric_statements:map[]]] f:receivers:map[f:otlp:map[.:map[] f:protocols:map[.:map[] f:grpc:map[] f:http:map[]]] f:prometheus:map[.:map[] f:config:map[.:map[] f:scrape_configs:map[]]] f:prometheus/self:map[.:map[] f:config:map[.:map[] f:scrape_configs:map[]]]] f:service:map[f:extensions:map[] f:pipelines:map[f:metrics/internal:map[.:map[] f:exporters:map[] f:receivers:map[]] f:metrics/otlp:map[.:map[] f:exporters:map[] f:processors:map[] f:receivers:map[]] f:metrics/scrape:map[.:map[] f:exporters:map[] f:processors:map[] f:receivers:map[]] f:traces:map[.:map[] f:exporters:map[] f:processors:map[] f:receivers:map[]]] f:telemetry:map[f:metrics:map[.:map[] f:readers:map[]]]]] f:mode:map[] f:networkPolicy:map[f:enabled:map[]] f:replicas:map[] f:resources:map[f:limits:map[f:cpu:map[] f:memory:map[]] f:requests:map[f:cpu:map[] f:memory:map[]]] f:targetAllocator:map[f:allowInsecureAuthSecrets:map[] f:enabled:map[] f:mtls:map[f:enabled:map[] f:useCertManager:map[]] f:prometheusCR:map[f:denyFSAccessThroughSMs:map[] f:enabled:map[] f:podMonitorSelector:map[] f:secretNamespaces:map[] f:serviceMonitorSelector:map[]] f:serviceAccount:map[]] f:volumeMounts:map[] f:volumes:map[]]] manager:monitoring operation:Apply time:2026-10-30T03:08:28Z] map[apiVersion:opentelemetry.io/v1beta1 fieldsType:FieldsV1 fieldsV1:map[f:status:map[.:map[] f:conditions:map[] f:image:map[] f:observedGeneration:map[] f:scale:map[.:map[] f:replicas:map[] f:selector:map[] f:statusReplicas:map[]] f:version:map[]]] manager:opentelemetry-operator operation:Update subresource:status time:2026-10-30T03:08:29Z]] name:data-science-collector namespace:redhat-ods-monitoring ownerReferences:[map[apiVersion:services.platform.opendatahub.io/v1alpha1 blockOwnerDeletion:true controller:true kind:Monitoring name:default-monitoring uid:f0164913-977c-4091-ab5c-df2c37ecec26]] resourceVersion:26298 uid:93c6fe5f-67d9-4a6c-b392-10c40496ae8c] spec:map[config:map[exporters:map[otlphttp/metrics-external:map[endpoint:http://metrics-external.example.svc:4318 tls:map[ins... Gomega truncated this representation as it exceeds 'format.MaxLength'. Consider having the object provide a custom 'GomegaStringer' representation or adjust the parameters in Gomega's 'format' package. Learn more here: https://onsi.github.io/gomega/#adjusting-output to match expression : (.spec.config.exporters | has("otlphttp/metrics-external") and has("otlphttp/traces-external")) and (.spec.config.exporters | has("prometheus") | not) and (.spec.config.exporters | has("otlp/tempo") | not) and (.spec.config.service.pipelines.metrics.exporters | length == 1 and contains(["otlphttp/metrics-external"])) and (.spec.config.service.pipelines.traces.exporters | length == 1 and contains(["otlphttp/traces-external"])) helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_metrics_only_with_storage monitoring_modes_test.go:38: Waiting for previous Prometheus pods to finish terminating monitoring_modes_test.go:38: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:166: Timed out after 600.001s. Mode 4 metrics-only: collector metrics pipeline must include built-in prometheus exporter The matcher passed to Eventually returned the following error: <*gojq.func1TypeError | 0x87d5c511710>: contains(["prometheus"]) cannot be applied to: null { name: "contains", v: nil, w: <[]interface {} | len:1, cap:1>["prometheus"], } helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_traces_only_with_storage monitoring_modes_test.go:39: Waiting for previous Prometheus pods to finish terminating helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_1_all_built-in_metrics_and_traces_with_storage monitoring_modes_test.go:40: Waiting for previous Prometheus pods to finish terminating monitoring_modes_test.go:40: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:166: Timed out after 600.001s. Mode 1: collector metrics/traces pipelines must include built-in prometheus and otlp/tempo exporters The matcher passed to Eventually returned the following error: <*gojq.func1TypeError | 0x87d5c4ec7e0>: contains(["prometheus"]) cannot be applied to: null { name: "contains", v: nil, w: <[]interface {} | len:1, cap:1>["prometheus"], } helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Metrics_ready helper_test.go:428: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:75: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:75: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Creation === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Configuration === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Prometheus_Self_ServiceMonitor_TLS_Fix helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_ownerReference_and_resourceVersion_stability === RUN TestMonitoring/Metrics_ready/Korrel8r === RUN TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources === RUN TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/reconciles_a_ready_TLS_query_service monitoring_korrel8r_test.go:46: same-namespace Korrel8r ingress probe denied: korrel8r-ingress-18e331c3b7b9cfdf === RUN TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/serves_bounded_authenticated_API_requests monitoring_korrel8r_test.go:47: Korrel8r graph contract is not established: no ready collector workload pod is available as a known object === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Basic_Traces_Configuration helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Trace_Ingestion_always_uses_TLS_via_gateway helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Metrics_Exporters helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Traces_Exporters helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === NAME TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector monitoring_test.go:425: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_signal-specific_collector_contract === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_replicas === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Metrics_TLS_is_always_enabled_for_Prometheus_exporter === RUN TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Monitoring_TLS_is_always_enabled_for_internal_telemetry_exporter === RUN TestMonitoring/Metrics_ready/Target_Allocator === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_deployment_with_metrics helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_Service_and_ConfigMap helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_RBAC_configuration helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Thanos_Querier === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_deployment_with_metrics === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_authorization_boundary === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_namespace_isolation monitoring_test.go:1026: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:1026: thanos route evidence persona=anonymous http_status=401 prometheus_status="" series_count=0 namespaces=[] monitoring_test.go:1026: thanos route evidence persona=restricted http_status=403 prometheus_status="" series_count=0 namespaces=[] monitoring_test.go:1026: Waiting for the authorized Thanos route to return namespace-scoped Prometheus data monitoring_test.go:1026: thanos route evidence persona=authorized http_status=200 prometheus_status="success" series_count=33 namespaces=[redhat-ods-monitoring] monitoring_test.go:1026: thanos route evidence persona=authorized-crafted-selector http_status=200 prometheus_status="success" series_count=33 namespaces=[redhat-ods-monitoring] monitoring_test.go:1026: thanos route evidence persona=authorized-post http_status=200 prometheus_status="success" series_count=33 namespaces=[redhat-ods-monitoring] monitoring_test.go:1026: thanos route evidence persona=authorized-duplicate-namespace-post http_status=400 prometheus_status="" series_count=0 namespaces=[] === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_Prometheus_NetworkPolicy_allows_Thanos_Querier_on_gRPC_port === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_baseline_fixture_—_record_rollout_identity_and_health monitoring_test.go:1030: ✓ THANOS BASELINE RECORDED (independent of Perses) • Component: Thanos Querier • Rollout identity: resourceVersion=45087 generation=1 • CR status: ThanosQuerierAvailable=True • Query readiness: Deployment thanos-querier-data-science-thanos-querier ready replicas up • PromQL endpoint: Route data-science-thanos-querier-route → data-science-thanos-querier-proxy (reencrypt TLS) • PromQL response: query="up" status=success series=33 === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP monitoring_test.go:1031: LIMITATION (Component=Thanos): independent version-boundary testing is UNSUPPORTED and no version boundary is exercised here. COO packages Perses, Thanos and MonitoringStack under a single operator version, so the Thanos operand version cannot be pinned or swapped independently. Current rollout identity: resourceVersion=45087 generation=1. Reported as SKIP (not PASS) so this is not mistaken for boundary coverage; revisit once operand pinning is available. === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_component_health_check_with_failure-attribution_diagnostics monitoring_test.go:1032: ✓ Thanos health check passed; failure-attribution diagnostics wired: FAILURE ATTRIBUTION | Component=Thanos | Rollout=resourceVersion=45087 generation=1 | Assertion=Deployment thanos-querier-data-science-thanos-querier ready replicas | Diagnostics: oc -n redhat-ods-monitoring logs deployment/thanos-querier-data-science-thanos-querier ; oc -n redhat-ods-monitoring describe deployment thanos-querier-data-science-thanos-querier ; oc -n redhat-ods-monitoring get thanosquerier data-science-thanos-querier -o yaml === RUN TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Integration_—_Thanos_⇄_Perses_datasource_wiring monitoring_test.go:1035: ✓ THANOS ⇄ PERSES DATASOURCE WIRING OK • Dependent dashboards: PersesDatasource data-science-prometheus-datasource → HTTPProxy → http://thanos-querier-data-science-thanos-querier.redhat-ods-monitoring.svc.cluster.local:10902 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_restricted_resource_configuration monitoring_test.go:2531: MonitoringStack CR is Available — prerequisites met for prometheus-namespace-proxy === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_secure_proxy_authentication monitoring_test.go:2532: MonitoringStack CR is Available — prerequisites met for prometheus-namespace-proxy === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Monitoring_proxy_TLS_profile_rollout monitoring_test.go:2533: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction monitoring_test.go:2534: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-tenant-metrics" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 20ms (resourceVersion="45652") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 18ms (found=false, err=namespaces "tests-monitoring-tenant-metrics-foreign" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 26ms (resourceVersion="45670") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 18ms (found=false, err=deployments.apps "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 49ms (resourceVersion="45696") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 14ms (found=false, err=services "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 24ms (resourceVersion="45703") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.coreos.com "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 20ms (resourceVersion="45713") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 17ms (found=false, err=serviceaccounts "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 23ms (resourceVersion="45726") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 17ms (found=false, err=rolebindings.rbac.authorization.k8s.io "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 57ms (resourceVersion="45742") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting /v1, Kind=ServiceAccount redhat-ods-monitoring/tenant-metrics-foreign-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 17ms (found=false, err=serviceaccounts "tenant-metrics-foreign-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating /v1, Kind=ServiceAccount redhat-ods-monitoring/tenant-metrics-foreign-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 17ms (resourceVersion="46180") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics-foreign/tenant-metrics-foreign-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: get completed in 15ms (found=false, err=rolebindings.rbac.authorization.k8s.io "tenant-metrics-foreign-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics-foreign/tenant-metrics-foreign-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction] resource attempt 1: create completed in 19ms (resourceVersion="46184") === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed monitoring_test.go:2535: Waiting for Prometheus StatefulSet and pods to be ready test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 16ms (found=false, err=namespaces "tests-monitoring-tenant-metrics" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 18ms (resourceVersion="46430") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-tenant-metrics-foreign" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-tenant-metrics-foreign test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 20ms (resourceVersion="46446") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 20ms (found=false, err=deployments.apps "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating apps/v1, Kind=Deployment tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 39ms (resourceVersion="46471") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 22ms (found=false, err=services "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Service tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 23ms (resourceVersion="46479") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.coreos.com "tenant-metrics-exporter" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating monitoring.coreos.com/v1, Kind=ServiceMonitor tests-monitoring-tenant-metrics/tenant-metrics-exporter test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 25ms (resourceVersion="46493") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 17ms (found=false, err=serviceaccounts "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=ServiceAccount tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 19ms (resourceVersion="46497") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=rolebindings.rbac.authorization.k8s.io "tenant-metrics-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-tenant-metrics/tenant-metrics-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 46ms (resourceVersion="46501") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-otlp-identity test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-otlp-identity" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-otlp-identity test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 18ms (resourceVersion="46502") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting /v1, Kind=ServiceAccount redhat-ods-monitoring/otlp-identity-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 17ms (found=false, err=serviceaccounts "otlp-identity-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating /v1, Kind=ServiceAccount redhat-ods-monitoring/otlp-identity-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 19ms (resourceVersion="46527") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: getting rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-otlp-identity/otlp-identity-viewer test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: get completed in 15ms (found=false, err=rolebindings.rbac.authorization.k8s.io "otlp-identity-viewer" not found) test_context_test.go:378: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: creating rbac.authorization.k8s.io/v1, Kind=RoleBinding tests-monitoring-otlp-identity/otlp-identity-viewer test_context_test.go:387: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed] resource attempt 1: create completed in 45ms (resourceVersion="46537") monitoring_test.go:2535: collector Prometheus scrape interval: 30s (Prometheus resource) monitoring_test.go:2535: collector warm-up pod=data-science-collector-collector-0 attributed=true pushStatus=200 pushFailed=false queryStatus=200 queryFailed=false seriesCount=5 monitoring_test.go:2535: collector warm-up pod=data-science-collector-collector-1 attributed=true pushStatus=200 pushFailed=false queryStatus=200 queryFailed=false seriesCount=9 monitoring_test.go:2535: observing OTLP delivery and tenant isolation for 1m40s (Prometheus scrape interval 30s) === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-0 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-1 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/OTLP_delivery === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/sender_attribution === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/forged_identity_sanitization === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/tenant_isolation === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_deployment === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_RBAC_configuration === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization monitoring_test.go:2538: Operator metrics RBAC resources verified test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: getting /v1, Kind=Namespace /redhat-ods-monitoring test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: get completed in 16ms (found=true, err=) test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: getting /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patching /v1, Kind=ConfigMap redhat-ods-monitoring/odh-observability-metrics-ca (resourceVersion="21250") test_context_test.go:413: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patch completed in 19ms (resourceVersion="48465") test_context_test.go:367: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics test_context_test.go:371: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: get completed in 16ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patching monitoring.rhobs/v1, Kind=ServiceMonitor redhat-ods-monitoring/odh-observability-metrics (resourceVersion="21251") test_context_test.go:413: [TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization] resource attempt 1: patch completed in 19ms (resourceVersion="21251") monitoring_test.go:2538: created operator ServiceMonitor in namespace redhat-ods-monitoring for Service in odh-observability-system === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthenticated_request_returns_401 monitoring_test.go:3112: operator metrics probe from redhat-ods-monitoring: status=401 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthorized_request_returns_403 monitoring_test.go:3122: operator metrics probe from redhat-ods-monitoring: status=403 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Authorized_request_returns_200 monitoring_test.go:3138: operator metrics probe from redhat-ods-monitoring: status=200 === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-alertmanager-ingress === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-prometheus-cluster-proxy-ingress === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-thanos-querier-ingress === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_and_TargetAllocator_NetworkPolicy_contract monitoring_test.go:2540: no running instrumented KServe workload exporting OTLP to the collector; only selector probes will run monitoring_test.go:2540: OTLP selector probes use namespace redhat-ods-monitoring (enrolled=false) monitoring_test.go:2540: OTLP selector probes use namespace collector-network-policy-18e332385848ec88 (enrolled=true) === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_status === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_finalize === RUN TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/OpenShift_project === NAME TestMonitoring/Metrics_ready helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_PV_Backend === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_TempoMonolithic_CR_Creation_with_PV_backend === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_MLflow_integration_RBAC helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_Tempo_trace_export_RBAC helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Tempo_gateway_tenant_and_authentication_wiring helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === NAME TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Traces_with_Cloud_Storage === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend test_context_test.go:367: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/s3-secret test_context_test.go:371: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: get completed in 16ms (found=false, err=secrets "s3-secret" not found) test_context_test.go:378: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/s3-secret test_context_test.go:387: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend] resource attempt 1: create completed in 18ms (resourceVersion="53598") === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend test_context_test.go:367: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/gcs-secret test_context_test.go:371: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: get completed in 16ms (found=false, err=secrets "gcs-secret" not found) test_context_test.go:378: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/gcs-secret test_context_test.go:387: [TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend] resource attempt 1: create completed in 21ms (resourceVersion="55229") 2026-10-30T04:00:54Z INFO override tempo configuration could potentially break the stack, use it carefully === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Instrumentation_CR_Traces_lifecycle === RUN TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Traces_Exporters_Reserved_Name_Validation === NAME TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Perses === RUN TestMonitoring/Perses/Group_8:_Perses monitoring_test.go:1953: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:1953: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_deployment_when_monitoring_is_managed === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_CR_configuration === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_NetworkPolicy_creation monitoring_thanos_network_policy_test.go:119: Perses: ready pod data-science-perses-0 has selector labels map[app.kubernetes.io/instance:data-science-perses app.kubernetes.io/managed-by:perses-operator app.kubernetes.io/name:data-science-perses] === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity monitoring_thanos_network_policy_test.go:119: querier: ready pod thanos-querier-data-science-thanos-querier-74b6964bcd-zvcm7 has selector labels map[app.kubernetes.io/managed-by:observability-operator app.kubernetes.io/part-of:ThanosQuerier] === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/prometheus monitoring_thanos_network_policy_test.go:119: prometheus: ready pod prometheus-data-science-monitoringstack-0 has selector labels map[app.kubernetes.io/instance:data-science-monitoringstack app.kubernetes.io/name:prometheus] monitoring_thanos_network_policy_test.go:86: Thanos ingress probe prometheus from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/perses monitoring_thanos_network_policy_test.go:119: perses: ready pod data-science-perses-0 has selector labels map[app.kubernetes.io/name:data-science-perses instance:data-science-perses] monitoring_thanos_network_policy_test.go:86: Thanos ingress probe perses from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/korrel8r monitoring_thanos_network_policy_test.go:119: korrel8r: ready pod korrel8r-5cbcc89b8f-5b48r has selector labels map[app.kubernetes.io/component:correlation app.kubernetes.io/name:korrel8r app.kubernetes.io/part-of:monitoring] monitoring_thanos_network_policy_test.go:86: Thanos ingress probe korrel8r from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/authenticated-proxy monitoring_thanos_network_policy_test.go:119: authenticated-proxy: ready pod data-science-thanos-querier-proxy-58d8d65bf9-hd7n8 has selector labels map[app:data-science-thanos-querier-proxy] monitoring_thanos_network_policy_test.go:86: Thanos ingress probe authenticated-proxy from redhat-ods-monitoring completed (allowed=true) === RUN TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/other-namespace-denied monitoring_thanos_network_policy_test.go:102: Thanos ingress probe untrusted from thanos-ingress-probe-18e332e452ed11f9 completed (allowed=false) === NAME TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity helper_test.go:369: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Creation_with_Traces === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Configuration === RUN TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_PersesDatasource_deployment_with_Prometheus === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_baseline_fixture_—_record_rollout_identity_and_health monitoring_test.go:1984: ✓ PERSES BASELINE RECORDED (independent of Thanos) • Component: Perses • Rollout identity: resourceVersion=60299 generation=2 • CR status: PersesAvailable=True • Dashboard config: .spec.config database+schemas present • Service ready: StatefulSet data-science-perses readyReplicas=1 • API: Pod data-science-perses-0 Running/Ready on :8080 === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP monitoring_test.go:1985: LIMITATION (Component=Perses): independent version-boundary testing is UNSUPPORTED and no version boundary is exercised here. COO packages Perses, Thanos and MonitoringStack under a single operator version, so the Perses operand version cannot be pinned or swapped independently. Current rollout identity: resourceVersion=60299 generation=2. Reported as SKIP (not PASS) so this is not mistaken for boundary coverage; revisit once operand pinning is available. === RUN TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_component_health_check_with_failure-attribution_diagnostics monitoring_test.go:1986: ✓ Perses health check passed; failure-attribution diagnostics wired: FAILURE ATTRIBUTION | Component=Perses | Rollout=resourceVersion=60299 generation=2 | Assertion=Pod data-science-perses-0 Running/Ready | Diagnostics: oc -n redhat-ods-monitoring logs data-science-perses-0 ; oc -n redhat-ods-monitoring describe pod data-science-perses-0 ; oc -n redhat-ods-monitoring get perses data-science-perses -o yaml === NAME TestMonitoring/Perses/Group_8:_Perses helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Webhooks === RUN TestMonitoring/Webhooks/Group_10:_Webhooks === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Setup monitoring_webhook_test.go:32: Setting up webhook tests: enabling monitoring and waiting for ready state monitoring_webhook_test.go:32: Webhook tests setup complete: monitoring is enabled and ready === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_namespace === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors monitoring_webhook_test.go:34: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:34: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: create completed in 18ms (resourceVersion="61737") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: create completed in 19ms (resourceVersion="61755") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors] resource attempt 1: create completed in 18ms (resourceVersion="61764") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels monitoring_webhook_test.go:35: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:35: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: get completed in 16ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: create completed in 18ms (resourceVersion="61844") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: create completed in 97ms (resourceVersion="61868") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels] resource attempt 1: create completed in 52ms (resourceVersion="61870") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection monitoring_webhook_test.go:36: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:36: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: create completed in 18ms (resourceVersion="61953") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: get completed in 18ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: create completed in 54ms (resourceVersion="61976") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection] resource attempt 1: create completed in 54ms (resourceVersion="61977") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update monitoring_webhook_test.go:37: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:37: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: get completed in 17ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: create completed in 20ms (resourceVersion="62017") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: get completed in 23ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: create completed in 21ms (resourceVersion="62035") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update] resource attempt 1: create completed in 20ms (resourceVersion="62041") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels monitoring_webhook_test.go:38: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:38: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: create completed in 20ms (resourceVersion="62091") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: get completed in 19ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: create completed in 22ms (resourceVersion="62109") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: get completed in 16ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels] resource attempt 1: create completed in 20ms (resourceVersion="62116") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace monitoring_webhook_test.go:39: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:39: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: get completed in 16ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: create completed in 19ms (resourceVersion="62177") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: get completed in 18ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: create completed in 21ms (resourceVersion="62195") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: get completed in 16ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace] resource attempt 1: create completed in 22ms (resourceVersion="62199") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace monitoring_webhook_test.go:40: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:40: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: create completed in 18ms (resourceVersion="62252") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: get completed in 16ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: create completed in 22ms (resourceVersion="62269") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: get completed in 18ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace] resource attempt 1: create completed in 22ms (resourceVersion="62278") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out monitoring_webhook_test.go:41: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:41: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: get completed in 16ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: create completed in 19ms (resourceVersion="62326") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: get completed in 22ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: create completed in 22ms (resourceVersion="62345") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out] resource attempt 1: create completed in 22ms (resourceVersion="62352") === RUN TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency monitoring_webhook_test.go:42: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_webhook_test.go:42: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: create completed in 21ms (resourceVersion="62392") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: create completed in 28ms (resourceVersion="62411") test_context_test.go:367: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency] resource attempt 1: create completed in 19ms (resourceVersion="62419") === RUN TestMonitoring/Usage_Logs_Collection === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_Collector_not_deployed_without_usage_logs_config helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_shared_UsageLogs_resources test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_shared_UsageLogs_resources] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-loki-shared-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_shared_UsageLogs_resources] resource attempt 1: get completed in 16ms (found=false, err=secrets "test-loki-shared-secret" not found) test_context_test.go:378: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_shared_UsageLogs_resources] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/test-loki-shared-secret test_context_test.go:387: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_shared_UsageLogs_resources] resource attempt 1: create completed in 20ms (resourceVersion="62969") === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_LokiStack_deployment === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_LokiStack_configuration === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_deployment === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_configuration === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_RBAC === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_network_policies === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract monitoring_usage_logs_test.go:76: ClusterLogForwarder contract is not established: ClusterLogForwarder CRD is unavailable === RUN TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector) test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: get completed in 15ms (found=false, err=secrets "test-loki-lifecycle-secret" not found) test_context_test.go:378: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: creating /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret test_context_test.go:387: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: create completed in 17ms (resourceVersion="65119") test_context_test.go:367: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: getting /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret test_context_test.go:371: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: get completed in 15ms (found=true, err=) test_context_test.go:404: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: patching /v1, Kind=Secret redhat-ods-monitoring/test-loki-lifecycle-secret (resourceVersion="65119") test_context_test.go:413: [TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector)] resource attempt 1: patch completed in 16ms (resourceVersion="65119") helper_test.go:428: Waiting for previous Prometheus pods to finish terminating === NAME TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection monitoring_phase_test.go:39: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Metrics_replicas monitoring_test.go:3387: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:3387: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Target_Allocator monitoring_test.go:3392: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3393: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses monitoring_test.go:3397: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3398: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses_datasource monitoring_test.go:3402: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3403: Waiting for previous Prometheus pods to finish terminating === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Prometheus_rules monitoring_test.go:3407: Waiting for Prometheus StatefulSet and pods to be ready === RUN TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled monitoring_test.go:3412: Waiting for previous Prometheus pods to finish terminating monitoring_test.go:3412: Waiting for Prometheus StatefulSet and pods to be ready monitoring_test.go:3413: Disabling monitoring to test webhook behavior when monitoring is disabled monitoring_test.go:3413: Monitoring disabled, proceeding to test webhook behavior monitoring_test.go:3413: Pre-test cleanup: removing tests-monitoring-injection namespace and monitors if they exist monitoring_test.go:3413: Pre-test cleanup completed test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: getting /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: get completed in 15ms (found=false, err=namespaces "tests-monitoring-injection" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: creating /v1, Kind=Namespace /tests-monitoring-injection test_context_test.go:387: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: create completed in 18ms (resourceVersion="79252") test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: getting monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: get completed in 15ms (found=false, err=podmonitors.monitoring.rhobs "test-podmonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: creating monitoring.rhobs/v1, Kind=PodMonitor tests-monitoring-injection/test-podmonitor test_context_test.go:387: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: create completed in 91ms (resourceVersion="79277") test_context_test.go:367: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: getting monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:371: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: get completed in 15ms (found=false, err=servicemonitors.monitoring.rhobs "test-servicemonitor" not found) test_context_test.go:378: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: creating monitoring.rhobs/v1, Kind=ServiceMonitor tests-monitoring-injection/test-servicemonitor test_context_test.go:387: [TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled] resource attempt 1: create completed in 47ms (resourceVersion="79280") monitoring_test.go:3413: Webhook correctly skipped injection when monitoring was disabled === RUN TestMonitoring/Disabled === RUN TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled === RUN TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled/Validate_monitoring_service_disabled --- FAIL: TestMonitoring (5160.69s) --- PASS: TestMonitoring/openshift-cert-manager-operator (0.15s) --- PASS: TestMonitoring/install-dependent-operators (0.00s) --- PASS: TestMonitoring/install-dependent-operators/tempo-product (36.73s) --- PASS: TestMonitoring/install-dependent-operators/opentelemetry-product (38.56s) --- PASS: TestMonitoring/install-dependent-operators/cluster-observability-operator (52.76s) --- PASS: TestMonitoring/Baseline_without_metrics (24.46s) --- PASS: TestMonitoring/Baseline_without_metrics/Base_Configuration (0.20s) --- PASS: TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration (0.20s) --- PASS: TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_Traces_default_content (0.10s) --- SKIP: TestMonitoring/Baseline_without_metrics/Base_Configuration/Group_1:_Base_Configuration/Test_active_operand_namespace_and_readiness_contract (0.03s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions (23.47s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions (23.47s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Metrics_negative_conditions (0.10s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Traces_negative_conditions (0.10s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Alerting_negative_conditions (0.10s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Perses_negative_conditions (0.10s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/NodeMetrics_negative_conditions (0.11s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/OpenTelemetry_negative_conditions (0.10s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Usage_Logs_negative_conditions (0.10s) --- PASS: TestMonitoring/Baseline_without_metrics/Negative_Conditions/Group_12:_Negative_Conditions/Partial_signal_conditions_preserve_independent_traces (22.75s) --- PASS: TestMonitoring/Baseline_without_metrics/Target_Allocator_absent (0.16s) --- PASS: TestMonitoring/Baseline_without_metrics/Thanos_Querier_absent (0.16s) --- PASS: TestMonitoring/Baseline_without_metrics/Perses_absent (0.15s) --- FAIL: TestMonitoring/Observability_Modes (2621.10s) --- FAIL: TestMonitoring/Observability_Modes/Group:_Observability_Modes (2621.10s) --- FAIL: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_metrics_exporters_only (620.79s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_traces_exporters_only (18.89s) --- FAIL: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_2_combined_exporters_only_distinct_names (620.79s) --- FAIL: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_metrics_only_with_storage (640.00s) --- PASS: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_4_traces_only_with_storage (57.21s) --- FAIL: TestMonitoring/Observability_Modes/Group:_Observability_Modes/Mode_1_all_built-in_metrics_and_traces_with_storage (663.27s) --- PASS: TestMonitoring/Metrics_ready (891.31s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack (77.33s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack (77.33s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Creation (6.25s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Metrics_MonitoringStack_CR_Configuration (0.04s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_Prometheus_Self_ServiceMonitor_TLS_Fix (0.36s) --- PASS: TestMonitoring/Metrics_ready/Metrics_&_MonitoringStack/Group_2:_Metrics_&_MonitoringStack/Test_ownerReference_and_resourceVersion_stability (70.68s) --- PASS: TestMonitoring/Metrics_ready/Korrel8r (10.49s) --- PASS: TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources (10.49s) --- PASS: TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/reconciles_a_ready_TLS_query_service (10.34s) --- SKIP: TestMonitoring/Metrics_ready/Korrel8r/Korrel8r_resources/serves_bounded_authenticated_API_requests (0.15s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector (100.33s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector (100.33s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations (72.83s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Basic_Traces_Configuration (16.70s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Trace_Ingestion_always_uses_TLS_via_gateway (20.74s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Metrics_Exporters (16.63s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_Configurations/Custom_Traces_Exporters (18.76s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_signal-specific_collector_contract (0.14s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_OpenTelemetry_Collector_replicas (16.55s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Metrics_TLS_is_always_enabled_for_Prometheus_exporter (0.23s) --- PASS: TestMonitoring/Metrics_ready/OpenTelemetry_Collector/Group_3:_OpenTelemetry_Collector/Test_Monitoring_TLS_is_always_enabled_for_internal_telemetry_exporter (0.23s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator (9.19s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator (9.19s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics (9.19s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_deployment_with_metrics (8.55s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_Service_and_ConfigMap (0.23s) --- PASS: TestMonitoring/Metrics_ready/Target_Allocator/Group_4:_Target_Allocator/With_Metrics/Test_Target_Allocator_RBAC_configuration (0.41s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier (8.26s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier (8.26s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_deployment_with_metrics (0.17s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_authorization_boundary (0.28s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_ThanosQuerier_route_namespace_isolation (0.95s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Test_Prometheus_NetworkPolicy_allows_Thanos_Querier_on_gRPC_port (0.13s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos (6.62s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_baseline_fixture_—_record_rollout_identity_and_health (0.37s) --- SKIP: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP (0.02s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Component_Isolation_—_Thanos/Test_Thanos_component_health_check_with_failure-attribution_diagnostics (6.24s) --- PASS: TestMonitoring/Metrics_ready/Thanos_Querier/Group_5:_Thanos_Querier/Integration_—_Thanos_⇄_Perses_datasource_wiring (0.11s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC (646.56s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC (646.56s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_restricted_resource_configuration (0.33s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Prometheus_secure_proxy_authentication (0.40s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Monitoring_proxy_TLS_profile_rollout (6.48s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Tenant_workload_metrics_namespace_restriction (64.49s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed (259.26s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-0 (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/collector_replica_warm-up/data-science-collector-collector-1 (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/OTLP_delivery (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/sender_attribution (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/forged_identity_sanitization (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/OTLP_metric_identity_cannot_be_spoofed/tenant_isolation (0.00s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_deployment (8.43s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Node_metrics_endpoint_RBAC_configuration (0.28s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization (6.23s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthenticated_request_returns_401 (0.26s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Unauthorized_request_returns_403 (0.34s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operator_metrics_authentication_and_authorization/Authorized_request_returns_200 (0.31s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle (8.46s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-alertmanager-ingress (2.12s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-prometheus-cluster-proxy-ingress (4.13s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Operand_NetworkPolicies_and_lifecycle/data-science-thanos-querier-ingress (2.13s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_and_TargetAllocator_NetworkPolicy_contract (262.13s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization (30.08s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace (0.23s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_status (0.17s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/namespace_finalize (0.16s) --- PASS: TestMonitoring/Metrics_ready/Networking_and_RBAC/Group_9:_Networking_and_RBAC/Collector_OTLP_namespace_authorization/OpenShift_project (0.21s) --- PASS: TestMonitoring/Traces_with_PV_Backend (129.43s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend (129.43s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_TempoMonolithic_CR_Creation_with_PV_backend (56.87s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_MLflow_integration_RBAC (2.46s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Collector_Tempo_trace_export_RBAC (18.78s) --- PASS: TestMonitoring/Traces_with_PV_Backend/Group_6:_Traces_with_PV_Backend/Test_Tempo_gateway_tenant_and_authentication_wiring (51.12s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage (281.22s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage (281.22s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage (260.37s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/S3_backend (130.24s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_TempoStack_CR_Creation_with_Cloud_Storage/GCS_backend (130.13s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Instrumentation_CR_Traces_lifecycle (16.49s) --- PASS: TestMonitoring/Traces_with_Cloud_Storage/Group_7:_Traces_with_Cloud_Storage/Test_Traces_Exporters_Reserved_Name_Validation (2.09s) --- PASS: TestMonitoring/Perses (101.21s) --- PASS: TestMonitoring/Perses/Group_8:_Perses (101.21s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle (6.59s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_deployment_when_monitoring_is_managed (6.26s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_CR_configuration (0.21s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Lifecycle/Test_Perses_NetworkPolicy_creation (0.12s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity (16.96s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/prometheus (2.13s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/perses (2.12s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/korrel8r (2.11s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/authenticated-proxy (2.12s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Test_ThanosQuerier_ingress_NetworkPolicy_connectivity/other-namespace-denied (8.20s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces (35.32s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Creation_with_Traces (18.63s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_Perses_Datasource_Configuration (16.52s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Perses_Datasource_with_Traces/Test_PersesDatasource_deployment_with_Prometheus (0.17s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses (0.37s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_baseline_fixture_—_record_rollout_identity_and_health (0.23s) --- SKIP: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_version_boundary_—_UNSUPPORTED_(no_operand_pinning),_reported_as_SKIP (0.02s) --- PASS: TestMonitoring/Perses/Group_8:_Perses/Component_Isolation_—_Perses/Test_Perses_component_health_check_with_failure-attribution_diagnostics (0.12s) --- PASS: TestMonitoring/Webhooks (70.49s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks (70.49s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Setup (18.46s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_namespace (0.05s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Label_value_enforcement_on_monitors (6.41s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitors_creation_with_custom_labels (6.48s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection (4.42s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update (6.58s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Monitor_label_injection_on_update_with_custom_labels (6.61s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_non-monitored_namespace (4.36s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_skips_explicitly_opted-out_namespace (6.37s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_respects_user_opt-out (4.36s) --- PASS: TestMonitoring/Webhooks/Group_10:_Webhooks/Webhook_idempotency (6.38s) --- PASS: TestMonitoring/Usage_Logs_Collection (648.36s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection (648.36s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_Collector_not_deployed_without_usage_logs_config (6.52s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Setup_shared_UsageLogs_resources (226.45s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_LokiStack_deployment (0.07s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_LokiStack_configuration (0.03s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_deployment (0.08s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_configuration (0.04s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_RBAC (0.09s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_Usage_Logs_Collector_network_policies (0.12s) --- SKIP: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Validate_optional_ClusterLogForwarder_and_Vector_contract (0.02s) --- PASS: TestMonitoring/Usage_Logs_Collection/Group_11:_Usage_Logs_Collection/Test_Usage_Logs_lifecycle_(LokiStack_+_Collector) (369.97s) --- PASS: TestMonitoring/Lifecycle (303.32s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle (303.32s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Metrics_replicas (41.87s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Target_Allocator (76.47s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses (59.37s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Perses_datasource (49.36s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Prometheus_rules (29.91s) --- PASS: TestMonitoring/Lifecycle/Group_13:_Lifecycle/Webhook_when_monitoring_disabled (46.34s) --- PASS: TestMonitoring/Disabled (0.41s) --- PASS: TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled (0.41s) --- PASS: TestMonitoring/Disabled/Disabled:_Monitoring_Service_Disabled/Validate_monitoring_service_disabled (0.41s) FAIL FAIL github.com/opendatahub-io/odh-observability/tests/e2e 5160.709s FAIL make: *** [Makefile:62: e2e-test] Error 1