gateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorself = <test_external_models.TestLegacyExternalModelMigration object at 0x7fa89e6ed9d0> def test_migration_sets_legacy_status_and_removes_networking(self): name = f"e2e-legacy-migration-{uuid.uuid4().hex[:8]}" secret_name = f"{name}-api-key" legacy_kind = "externalmodels.maas.opendatahub.io" legacy_resource_name = f"maas-{name}" expected_message = f"Model migrated to inference.opendatahub.io ExternalModel: {name}" deadline = time.monotonic() + 180 try: _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": {"name": secret_name, "namespace": MODEL_NAMESPACE}, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Seed one legacy networking child so this test proves teardown ran, # even if IPP migrates the model before the legacy reconciler has # time to create the full networking set. _apply_cr({ "apiVersion": "v1", "kind": "Service", "metadata": {"name": legacy_resource_name, "namespace": MODEL_NAMESPACE}, "spec": { "type": "ExternalName", "externalName": EXTERNAL_ENDPOINT, "ports": [{"name": "https", "port": 443}], }, }) > _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": name, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "targetModel": TARGET_MODEL, "endpoint": EXTERNAL_ENDPOINT, "credentialRef": {"name": secret_name}, }, }) test/e2e/tests/test_external_models.py:485: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'ExternalModel', 'metadata': {'name': 'e2e-legacy-migration-eab...el': 'gpt-3.5-turbo', 'endpoint': 'httpbingo.org', 'credentialRef': {'name': 'e2e-legacy-migration-eabf40d2-api-key'}}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for ExternalModel llm/e2e-legacy-migration-eabf40d2: Error from server (Forbidden): error when creating "STDIN": admission webhook "vexternalmodel.kb.io" denied the request: maas.opendatahub.io ExternalModel is deprecated; create inference.opendatahub.io ExternalProvider and ExternalModel instead (see the External Model setup guide in the MaaS documentation) test/e2e/tests/test_helper.py:894: RuntimeErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutErrorgateway_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IkN0QUgzN3Q4YkdNbnVhZjkzSEVQWXE1MG9lY2dxeEZxM2NDSzI1MTY3QWcifQ.e...KQi31gYNLoTQ7CdGCrBTC1bIpCH7N-H9jNMsz2a1keE21WDg_MuHIlVsY6cqZCaOFeJlRW1SOnRIN9RlA', 'Content-Type': 'application/json'} api_keys_base_url = 'https://maas.apps.4ec91afccb26205cb961.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' @pytest.fixture(scope="module") def external_models_setup(gateway_url, headers, api_keys_base_url): """ Create a single ExternalModel CR, MaaSModelRef, AuthPolicy, and Subscription pointing to an external endpoint. Cleanup after tests. """ log.info(f"Setting up external model test fixture (endpoint: {EXTERNAL_ENDPOINT})...") try: # Create a dummy secret (ExternalModel requires credentialRef). # The apikey-injection plugin's secret store only watches Secrets carrying # this label; without it credential lookups silently fail with a 500 at # request time (masked by tests that only assert non-401/403). _apply_cr({ "apiVersion": "v1", "kind": "Secret", "metadata": { "name": f"{EXTERNAL_MODEL_NAME}-api-key", "namespace": MODEL_NAMESPACE, "labels": {"inference.llm-d.ai/ipp-managed": "true"}, }, "type": "Opaque", "stringData": {"api-key": "e2e-test-key"}, }) # Create ExternalProvider CR (inference.opendatahub.io — canonical) _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalProvider", "metadata": {"name": EXTERNAL_PROVIDER_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "provider": "openai", "endpoint": EXTERNAL_ENDPOINT, "auth": { "type": "simple", "secretRef": {"name": f"{EXTERNAL_MODEL_NAME}-api-key"}, }, }, }) # Create ExternalModel CR (inference.opendatahub.io — canonical). This is # what the IPP model-provider-resolver plugin watches to populate its # model->provider store, and what maas-controller's externalModelHandler # tries first when reconciling the MaaSModelRef. _apply_cr({ "apiVersion": "inference.opendatahub.io/v1alpha1", "kind": "ExternalModel", "metadata": {"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}, "spec": { "externalProviderRefs": [ { "ref": {"name": EXTERNAL_PROVIDER_NAME}, "targetModel": TARGET_MODEL, "apiFormat": "openai-chat", "path": "/v1/chat/completions", }, ], }, }) # Create MaaSModelRef _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "annotations": { "maas.opendatahub.io/endpoint": EXTERNAL_ENDPOINT, "maas.opendatahub.io/provider": "openai", }, }, "spec": { "modelRef": {"kind": "ExternalModel", "name": EXTERNAL_MODEL_NAME}, }, }) # Create MaaSAuthPolicy _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": EXTERNAL_AUTH_POLICY, "namespace": _subscription_ns()}, "spec": { "modelRefs": [{"name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create MaaSSubscription _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": EXTERNAL_SUBSCRIPTION, "namespace": _subscription_ns()}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [ { "name": EXTERNAL_MODEL_NAME, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 10000, "window": "1h"}], }, ], }, }) # Wait for CRs to reconcile _wait_for_maas_auth_policy_phase(EXTERNAL_AUTH_POLICY, namespace=_subscription_ns()) _wait_for_maas_subscription_phase(EXTERNAL_SUBSCRIPTION, namespace=_subscription_ns()) # MaaSAuthPolicy phase Active only proves the gateway's fixed-size, # shared Kuadrant AuthPolicy (maas-gateway-auth) is Enforced — that # singleton doesn't change when a MaaSAuthPolicy is added, so it says # nothing about this ExternalModel's own HTTPRoute. Wait for the # HTTPRoute itself to be Accepted by the gateway, otherwise # TestExternalModelAuth's requests can race the data plane and get a # plain 404 ("no route") instead of a proper 401/403. > _wait_for_httproute_accepted(EXTERNAL_MODEL_HTTPROUTE_NAME, namespace=MODEL_NAMESPACE) test/e2e/tests/test_external_models.py:257: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-external-model', namespace = 'llm', timeout = 60 def _wait_for_httproute_accepted(name, namespace=MODEL_NAMESPACE, timeout=60): """Wait for an HTTPRoute to be Accepted by at least one parent Gateway. The HTTPRoute object existing in the API server only proves the reconciler created it — the Gateway controller still needs to program it into the data plane before real HTTP traffic will match it. Requests sent before that happens race the gateway and get a plain 404 ("no route"), which is easy to mistake for an auth failure since callers usually assert on 401/403. Waiting on the gateway-level AuthPolicy Enforced condition does NOT prove this route is ready — that policy is a fixed-size singleton unrelated to any single HTTPRoute. Args: name: Name of the HTTPRoute namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The HTTPRoute CR dict once Accepted Raises: TimeoutError: If no parent reports Accepted=True within timeout """ deadline = time.time() + timeout log.info(f"Waiting for HTTPRoute {namespace}/{name} to be Accepted (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("httproute", name, namespace) if cr: parents = (cr.get("status") or {}).get("parents") or [] for parent in parents: if any( c.get("type") == "Accepted" and c.get("status") == "True" for c in parent.get("conditions") or [] ): log.info(f"HTTPRoute {namespace}/{name} is Accepted") return cr log.debug(f"HTTPRoute {namespace}/{name}: no parent Accepted=True yet ({len(parents)} parent(s))") time.sleep(2) cr = _get_cr("httproute", name, namespace) parents = (cr.get("status") or {}).get("parents") if cr else None > raise TimeoutError( f"HTTPRoute {namespace}/{name} did not report Accepted=True within {timeout}s " f"(parents={parents!r})" ) E TimeoutError: HTTPRoute llm/e2e-external-model did not report Accepted=True within 60s (parents=None) test/e2e/tests/test_helper.py:2147: TimeoutError/workspace/source/test/e2e/tests/test_gateway_filter_chain.py:296: default tenant runs praxis-extproc; ai-gateway-controller owns its EnvoyFilter/workspace/source/test/e2e/tests/test_gateway_filter_chain.py:300: default tenant runs praxis-extproc; ai-gateway-controller owns its EnvoyFilter/workspace/source/test/e2e/tests/test_gateway_filter_chain.py:265: default tenant runs praxis-extproc; ai-gateway-controller owns its EnvoyFilter/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:235: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:266: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)/workspace/source/test/e2e/tests/test_subscription.py:828: no Kuadrant WasmPlugin for gateway maas-default-gateway/workspace/source/test/e2e/tests/test_subscription.py:1615: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7fa89e6c5550> @pytest.mark.serial def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid model (will be Active) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) cr = _wait_for_maas_subscription_phase(subscription_name, "Active") # This test overrides controller-owned status to exercise the gateway # rejection path. Let initial TRLP/discovery reconciliation settle first, # and run serially so a queued reconcile from parallel test activity does # not overwrite the injected Failed phase before the inference request. _wait_for_subscription_inference_ready( subscription_name, MODEL_REF, namespace=ns, model_namespace=MODEL_NAMESPACE, ) # Verify it starts as Active phase = cr.get("status", {}).get("phase") log.info(f"Initial phase: {phase}") assert phase == "Active", f"Expected Active initially, got {phase}" # Create API key while Active api_key = _create_api_key( oc_token, name="failed-sub-test", subscription=subscription_name ) # Verify inference works while Active log.info("Testing inference while Active...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) assert r.status_code == 200, f"Expected 200 while Active, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference works with Active subscription") # Manually patch subscription to Failed phase import subprocess import json from datetime import datetime log.info("Manually patching subscription to Failed phase...") patch_data = { "status": { "phase": "Failed", "conditions": [ { "type": "Ready", "status": "False", "reason": "Failed", "message": "Subscription failed", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") } ], "modelRefStatuses": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Model failed" } ] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data), "-o", "json", ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch to Failed phase: {result.stderr}" # Read the object returned by the patch itself; a second API request # leaves extra time for any queued controller reconciliation to win. patched_cr = json.loads(result.stdout) phase = patched_cr.get("status", {}).get("phase") assert phase == "Failed", f"Expected Failed phase after patch, got {phase}" log.info("✅ Subscription patched to Failed phase") # Test inference with Failed subscription - should be rejected by OPA log.info("Testing inference with Failed subscription...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) log.info(f"Response: status={r.status_code}, body={r.text[:200]}") # Failed phase should be rejected by OPA rule (403 or error message) if r.status_code == 200: > assert "denied" in r.text.lower() or "access" in r.text.lower(), f"Expected access denied message, got: {r.text[:200]}" E AssertionError: Expected access denied message, got: {"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_tra E assert ('denied' in '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"today it "}]}' or 'access' in '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"today it "}]}') E + where '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"today it "}]}' = <built-in method lower of str object at 0x7fa89e56cea0>() E + where <built-in method lower of str object at 0x7fa89e56cea0> = '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"Today it "}]}'.lower E + where '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"Today it "}]}' = <Response [200]>.text E + and '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"today it "}]}' = <built-in method lower of str object at 0x7fa89e56d290>() E + where <built-in method lower of str object at 0x7fa89e56d290> = '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"Today it "}]}'.lower E + where '{"id":"cmpl-d80f5d89-dcae-5d85-bc0b-a2ae56f7d8f3","created":1793566696,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"stop","text":"Today it "}]}' = <Response [200]>.text test/e2e/tests/test_subscription.py:3081: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:401: default payload-processing is praxis-extproc; MaasTenantConfig.spec.payloadProcessing.resources is MaaS IPP-only/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:375: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:384: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:399: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:409: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:423: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:359: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true