/workspace/source/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gatewayrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_deduplication_same_model_multiple_refs>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_user_token_returns_all_models>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w2-5d74b9', namespace = 'e2e-models-e2e-worker-w2-5d74b9' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w2-5d74b9/distinct-w2-5d74b9 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_user_token_returns_all_models>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w2-5d74b9', namespace = 'e2e-models-e2e-worker-w2-5d74b9' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w2-5d74b9/distinct-w2-5d74b9 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_response_schema_and_metadata>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w3-b4708a', namespace = 'e2e-models-e2e-worker-w3-b4708a' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w3-b4708a/distinct-w3-b4708a did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_response_schema_and_metadata>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w3-b4708a', namespace = 'e2e-models-e2e-worker-w3-b4708a' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w3-b4708a/distinct-w3-b4708a did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_empty_subscription_header_value>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w1-6796b1', namespace = 'e2e-models-e2e-worker-w1-6796b1' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w1-6796b1/distinct-w1-6796b1 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_empty_subscription_header_value>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w1-6796b1', namespace = 'e2e-models-e2e-worker-w1-6796b1' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w1-6796b1/distinct-w1-6796b1 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_api_key_with_deleted_subscription_403>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w0-36bbac', namespace = 'e2e-models-e2e-worker-w0-36bbac' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w0-36bbac/distinct-w0-36bbac did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_models_context' for <Function test_api_key_with_deleted_subscription_403>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, ( # The baseline model is bootstrapped (and waited on) once per worker # session; re-ensuring it here heals a backend that drifted Unhealthy # during the run. gw3 hit exactly that: simulator-w3-78d5ff was # Unhealthy when this class started, so setup_class failed whichever # tests loadgroup had scheduled last on that worker. context.model_ref, context.distinct_model_ref, context.distinct_model_2_ref, ), ) test/e2e/tests/test_models_endpoint.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w0-36bbac', namespace = 'e2e-models-e2e-worker-w0-36bbac' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w0-36bbac/distinct-w0-36bbac did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutError/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_networkpolicy.py:119: AIGC praxis NetworkPolicy does not include OTLP 4317 egress (MaaS Go IPP bundle only)/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)self = <test_subscription.TestStatusReporting object at 0x7fd856ecd990> request = <SubRequest '_worker_status_models' for <Function test_subscription_failed_status_with_missing_model>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w3-b4708a', namespace = 'e2e-models-e2e-worker-w3-b4708a' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w3-b4708a/distinct-w3-b4708a did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorself = <test_subscription.TestStatusReporting object at 0x7fd856ecd990> request = <SubRequest '_worker_status_models' for <Function test_subscription_failed_status_with_missing_model>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w3-b4708a', namespace = 'e2e-models-e2e-worker-w3-b4708a' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w3-b4708a/distinct-w3-b4708a did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorself = <test_subscription.TestStatusReporting object at 0x7f8d91490cd0> request = <SubRequest '_worker_status_models' for <Function test_subscription_active_status_with_valid_model>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w0-36bbac', namespace = 'e2e-models-e2e-worker-w0-36bbac' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w0-36bbac/distinct-w0-36bbac did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorself = <test_subscription.TestStatusReporting object at 0x7f8d91490cd0> request = <SubRequest '_worker_status_models' for <Function test_subscription_active_status_with_valid_model>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w0-36bbac', namespace = 'e2e-models-e2e-worker-w0-36bbac' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w0-36bbac/distinct-w0-36bbac did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorself = <test_subscription.TestStatusReporting object at 0x7f98ce28da10> request = <SubRequest '_worker_status_models' for <Function test_subscription_degraded_status_with_partial_models>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorself = <test_subscription.TestStatusReporting object at 0x7f98ce28da10> request = <SubRequest '_worker_status_models' for <Function test_subscription_degraded_status_with_partial_models>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorself = <test_subscription.TestStatusReporting object at 0x7f7c1ae01890> request = <SubRequest '_worker_status_models' for <Function test_subscription_status_transitions_on_model_deletion>> @pytest.fixture(scope="class", autouse=True) def _worker_status_models(self, request): """Provision the optional models used by status-reporting tests.""" from worker_tenant_fixtures import ensure_worker_models, serial_only_selection if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription.py:2531: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w1-6796b1', namespace = 'e2e-models-e2e-worker-w1-6796b1' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w1-6796b1/distinct-w1-6796b1 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_returns_accessible_subscriptions>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w2-5d74b9', namespace = 'e2e-models-e2e-worker-w2-5d74b9' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w2-5d74b9/distinct-w2-5d74b9 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_returns_accessible_subscriptions>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w2-5d74b9', namespace = 'e2e-models-e2e-worker-w2-5d74b9' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w2-5d74b9/distinct-w2-5d74b9 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_unauthenticated_returns_401[subscriptions-for-model]>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w3-b4708a', namespace = 'e2e-models-e2e-worker-w3-b4708a' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w3-b4708a/distinct-w3-b4708a did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_unauthenticated_returns_401[subscriptions-for-model]>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w3-b4708a', namespace = 'e2e-models-e2e-worker-w3-b4708a' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w3-b4708a/distinct-w3-b4708a did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_model_ref_display_name_and_description_enriched>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w0-36bbac', namespace = 'e2e-models-e2e-worker-w0-36bbac' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w0-36bbac/distinct-w0-36bbac did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_model_ref_display_name_and_description_enriched>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w0-36bbac', namespace = 'e2e-models-e2e-worker-w0-36bbac' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w0-36bbac/distinct-w0-36bbac did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_unknown_model_returns_empty>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_unknown_model_returns_empty>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w4-a2f984', namespace = 'e2e-models-e2e-worker-w4-a2f984' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w4-a2f984/distinct-w4-a2f984 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutErrorrequest = <SubRequest '_worker_subscription_list_context' for <Function test_omits_subscription_with_no_authorized_models>> @pytest.fixture(scope="module", autouse=True) def _worker_subscription_list_context(request): """Bind subscription-list tests to worker-owned MaaS resources.""" from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return context = request.getfixturevalue("worker_tenant_context") > ensure_worker_models( context, (context.distinct_model_ref, context.distinct_model_2_ref), ) test/e2e/tests/test_subscription_list_endpoints.py:142: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/worker_tenant_fixtures.py:344: in ensure_worker_models _wait_for_model_ready( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'distinct-w1-6796b1', namespace = 'e2e-models-e2e-worker-w1-6796b1' timeout = 180 def _wait_for_model_ready(model_ref, namespace=MODEL_NAMESPACE, timeout=60): """Wait for MaaSModelRef to reach Ready phase. Args: model_ref: Name of the MaaSModelRef namespace: Namespace (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Returns: The MaaSModelRef CR dict when Ready Raises: TimeoutError: If MaaSModelRef doesn't reach Ready within timeout """ deadline = time.time() + timeout log.info(f"Waiting for MaaSModelRef {namespace}/{model_ref} to reach phase 'Ready' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasmodelref", model_ref, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") endpoint = status.get("endpoint") if phase == "Ready" and endpoint: log.info(f"MaaSModelRef {namespace}/{model_ref} is Ready with endpoint: {endpoint}") return cr log.debug(f"MaaSModelRef {namespace}/{model_ref}: phase={phase}, endpoint={endpoint or 'none'}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasmodelref", model_ref, namespace) status = cr.get("status", {}) if cr else {} conditions = status.get("conditions", []) > raise TimeoutError( f"MaaSModelRef {namespace}/{model_ref} did not reach Ready within {timeout}s " f"(current: phase={status.get('phase')}, endpoint={status.get('endpoint')}, " f"conditions={[c.get('type') + '=' + str(c.get('status')) for c in conditions]})" ) E TimeoutError: MaaSModelRef e2e-models-e2e-worker-w1-6796b1/distinct-w1-6796b1 did not reach Ready within 180s (current: phase=Pending, endpoint=None, conditions=['GovernanceAttached=False', 'RuntimeReady=True', 'ModelIdentityUnique=True', 'Ready=False']) test/e2e/tests/test_helper.py:2046: TimeoutError/workspace/source/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:268: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:341: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:396: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:434: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_conflicts.py:367: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-cfl-2-62fc48/e2e-cfl-same-a7c3b5 status.resolvedModelAlias='e2e-cfl-same-a7c3b5' != 'e2e-cfl-same-a7c3b5-two'/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:227: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:199: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:307: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:251: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_shared.py:165: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-shr-user-3d0e7c/e2e-shr-shared-2004c8-e2e-extmod-a-ed8266 status.resolvedModelAlias='e2e-shr-shared-2004c8' != 'e2e-shr-shared-2004c8-client'/workspace/source/test/e2e/tests/test_tenant_external_model_shared.py:165: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-shr-user-45d763/e2e-shr-quota-ff63e9-e2e-extmod-a-05bee8 status.resolvedModelAlias='e2e-shr-quota-ff63e9' != 'e2e-shr-quota-ff63e9-client'/workspace/source/test/e2e/tests/test_tenant_external_model_shared.py:165: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-shr-user-3d0e7c/e2e-shr-auth-9db7f0-e2e-extmod-a-ed8266 status.resolvedModelAlias='e2e-shr-auth-9db7f0' != 'e2e-shr-auth-9db7f0-client'