{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"aiguardrail","msg":"NemoGuardrails CRD not installed; AIGuardrail provider watch disabled","crd":"trustyai.opendatahub.io/v1alpha1, Kind=NemoGuardrails","effect":"every AIGuardrail is refused until TrustyAI is installed; policies recover on their own, but until this controller restarts provider edits are picked up at the resync interval rather than immediately"}
{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"setup","msg":"starting ai-gateway-controller","manifestPath":"/config/manifests/external-model/overlays/odh","image":"quay.io/opendatahub/odh-praxis-extproc:odh-stable"}
{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-10-31T18:29:29Z","msg":"starting server","name":"health probe","addr":"[::]:8081"}
{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-metrics-server/metrics-certs/tls.crt","key":"/tmp/k8s-metrics-server/metrics-certs/tls.key"}
{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8443","secure":true}
{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-metrics-server/metrics-certs/tls.crt","key":"/tmp/k8s-metrics-server/metrics-certs/tls.key","interval":10}
{"level":"info","ts":"2026-10-31T18:29:29Z","logger":"leaderelection","msg":"Attempting to acquire leader lease...","lock":"opendatahub/ai-gateway-controller-leader.opendatahub.io"}
{"level":"info","ts":"2026-10-31T18:29:54Z","logger":"leaderelection","msg":"Successfully acquired lease","lock":"opendatahub/ai-gateway-controller-leader.opendatahub.io"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1.ConfigMap"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 MaasTenantConfig]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 AITenant]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 AITenant]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *unstructured.Unstructured[gateway.networking.k8s.io/v1beta1 ReferenceGrant]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"aiguardrail","controllerGroup":"aigateway.opendatahub.io","controllerKind":"AIGuardrail","source":"kind source: *v1.Namespace"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1.PartialObjectMetadata[v1 Secret]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *v1.PartialObjectMetadata[v1 Secret]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 MaasTenantConfig]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalProvider"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1.PartialObjectMetadata[v1 Secret]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *v1alpha1.ExternalModel"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 MaasTenantConfig]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 AITenant]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","source":"kind source: *unstructured.Unstructured[gateway.networking.k8s.io/v1 HTTPRoute]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *unstructured.Unstructured[dscinitialization.opendatahub.io/v2 DSCInitialization]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"aiguardrail","controllerGroup":"aigateway.opendatahub.io","controllerKind":"AIGuardrail","source":"kind source: *v1alpha1.AIGuardrail"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 MaasTenantConfig]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *unstructured.Unstructured[maas.opendatahub.io/v1alpha1 AITenant]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1.PartialObjectMetadata[v1 Secret]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1.PartialObjectMetadata[v1 ConfigMap]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *v1.PartialObjectMetadata[v1 Service]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting EventSource","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","source":"kind source: *unstructured.Unstructured[gateway.networking.k8s.io/v1 HTTPRoute]"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting Controller","controller":"aiguardrail","controllerGroup":"aigateway.opendatahub.io","controllerKind":"AIGuardrail"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting workers","controller":"aiguardrail","controllerGroup":"aigateway.opendatahub.io","controllerKind":"AIGuardrail","worker count":1}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting Controller","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting workers","controller":"externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","worker count":1}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting Controller","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting workers","controller":"consumer-portal","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","worker count":1}
{"level":"info","ts":"2026-10-31T18:29:54Z","logger":"consumer-portal","msg":"default AITenant has no OIDC issuer; OpenShift OAuth mode is not supported yet, removing the auth proxy","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"}}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting Controller","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting workers","controller":"externalprovider","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalProvider","worker count":1}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting Controller","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig"}
{"level":"info","ts":"2026-10-31T18:29:54Z","msg":"Starting workers","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","worker count":1}
{"level":"info","ts":"2026-10-31T18:29:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:31:24Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress","gatewayName":"e2e-ait-bbf5caaf"}
{"level":"info","ts":"2026-10-31T18:31:27Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress","gatewayName":"e2e-ait-bbf5caaf"}
{"level":"info","ts":"2026-10-31T18:31:29Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-e4c133ca"},"tenantID":"e2e-ait-e4c133ca","namespace":"openshift-ingress","gatewayName":"e2e-ait-e4c133ca"}
{"level":"info","ts":"2026-10-31T18:31:31Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"},"tenantID":"e2e-ait-8d3cca1f","namespace":"openshift-ingress","gatewayName":"e2e-ait-8d3cca1f"}
{"level":"info","ts":"2026-10-31T18:31:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-1f4b473d"},"tenantID":"e2e-migrate-1f4b473d","namespace":"openshift-ingress","gatewayName":"e2e-migrate-1f4b473d-gw"}
{"level":"info","ts":"2026-10-31T18:31:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress","gatewayName":"e2e-ait-bbf5caaf"}
{"level":"info","ts":"2026-10-31T18:31:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-e4c133ca"},"tenantID":"e2e-ait-e4c133ca","namespace":"openshift-ingress","gatewayName":"e2e-ait-e4c133ca"}
{"level":"info","ts":"2026-10-31T18:31:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"},"tenantID":"e2e-ait-8d3cca1f","namespace":"openshift-ingress","gatewayName":"e2e-ait-8d3cca1f"}
{"level":"info","ts":"2026-10-31T18:31:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"tenantID":"e2e-derive-3c7a8371","namespace":"openshift-ingress","gatewayName":"e2e-derive-3c7a8371"}
{"level":"info","ts":"2026-10-31T18:31:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-1f4b473d"},"tenantID":"e2e-migrate-1f4b473d","namespace":"openshift-ingress","gatewayName":"e2e-migrate-1f4b473d-gw"}
{"level":"info","ts":"2026-10-31T18:31:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress","gatewayName":"e2e-ait-bbf5caaf"}
{"level":"info","ts":"2026-10-31T18:31:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"tenantID":"e2e-derive-3c7a8371","namespace":"openshift-ingress","gatewayName":"e2e-derive-3c7a8371"}
{"level":"info","ts":"2026-10-31T18:31:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"tenantID":"e2e-derive-3c7a8371","namespace":"openshift-ingress","gatewayName":"e2e-derive-3c7a8371"}
{"level":"info","ts":"2026-10-31T18:31:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-e4c133ca"},"tenantID":"e2e-ait-e4c133ca","namespace":"openshift-ingress","gatewayName":"e2e-ait-e4c133ca"}
{"level":"info","ts":"2026-10-31T18:31:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"},"tenantID":"e2e-ait-8d3cca1f","namespace":"openshift-ingress","gatewayName":"e2e-ait-8d3cca1f"}
{"level":"info","ts":"2026-10-31T18:32:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-e4c133ca"},"tenantID":"e2e-ait-e4c133ca","namespace":"openshift-ingress","gatewayName":"e2e-ait-e4c133ca"}
{"level":"info","ts":"2026-10-31T18:32:04Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"},"tenantID":"e2e-ait-8d3cca1f","namespace":"openshift-ingress","gatewayName":"e2e-ait-8d3cca1f"}
{"level":"info","ts":"2026-10-31T18:32:05Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-e4c133ca"}}
{"level":"info","ts":"2026-10-31T18:32:08Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress","gatewayName":"e2e-ait-bbf5caaf"}
{"level":"info","ts":"2026-10-31T18:32:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress","gatewayName":"e2e-ait-bbf5caaf"}
{"level":"info","ts":"2026-10-31T18:32:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-1f4b473d"},"tenantID":"e2e-migrate-1f4b473d","namespace":"openshift-ingress","gatewayName":"e2e-migrate-1f4b473d-gw"}
{"level":"info","ts":"2026-10-31T18:32:15Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-e4c133ca"},"tenantID":"e2e-ait-e4c133ca","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-31T18:32:16Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"}}
{"level":"info","ts":"2026-10-31T18:32:17Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-1f4b473d"}}
{"level":"info","ts":"2026-10-31T18:32:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"tenantID":"e2e-derive-3c7a8371","namespace":"openshift-ingress","gatewayName":"e2e-derive-3c7a8371"}
{"level":"info","ts":"2026-10-31T18:32:24Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"}}
{"level":"info","ts":"2026-10-31T18:32:26Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-bbf5caaf"},"tenantID":"e2e-ait-bbf5caaf","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-31T18:32:29Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-migrate-1f4b473d"},"tenantID":"e2e-migrate-1f4b473d","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-31T18:32:30Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"}}
{"level":"info","ts":"2026-10-31T18:32:31Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"}}
{"level":"info","ts":"2026-10-31T18:32:34Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:32:35Z","logger":"tenant","msg":"owning AITenant changed (uid, ownership, readiness, generation, or gatewayRef) before praxis-extproc apply; will re-resolve","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"}}
{"level":"info","ts":"2026-10-31T18:32:37Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"tenantID":"e2e-derive-3c7a8371","namespace":"openshift-ingress"}
{"level":"error","ts":"2026-10-31T18:32:37Z","msg":"Reconciler error","controller":"maastenantconfig","controllerGroup":"maas.opendatahub.io","controllerKind":"MaasTenantConfig","MaasTenantConfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"namespace":"ai-tenant-e2e-derive-3c7a8371","name":"default-tenant","reconcileID":"7c2b7abf-0bee-4261-a0c1-b94619b8a93a","error":"Operation cannot be fulfilled on maastenantconfigs.maas.opendatahub.io \"default-tenant\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:497\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:440\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/internal/controller/controller.go:315"}
{"level":"info","ts":"2026-10-31T18:32:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:32:41Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-derive-3c7a8371"},"tenantID":"e2e-derive-3c7a8371","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-31T18:32:43Z","logger":"tenant","msg":"praxis-extproc resources cleaned up for deleted tenant","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-ait-8d3cca1f"},"tenantID":"e2e-ait-8d3cca1f","namespace":"openshift-ingress"}
{"level":"info","ts":"2026-10-31T18:32:46Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-6279f5"},"tenantID":"e2e-worker-w4-6279f5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-6279f5"}
{"level":"info","ts":"2026-10-31T18:32:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:32:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-6279f5"},"tenantID":"e2e-worker-w4-6279f5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-6279f5"}
{"level":"info","ts":"2026-10-31T18:32:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:32:55Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:32:57Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-6279f5"},"tenantID":"e2e-worker-w4-6279f5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-6279f5"}
{"level":"info","ts":"2026-10-31T18:33:00Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:33:02Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:33:05Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-6279f5"},"tenantID":"e2e-worker-w4-6279f5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-6279f5"}
{"level":"info","ts":"2026-10-31T18:33:07Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:33:09Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:33:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:33:14Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:33:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:33:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:33:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:33:23Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:33:26Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-6279f5"},"tenantID":"e2e-worker-w4-6279f5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-6279f5"}
{"level":"info","ts":"2026-10-31T18:33:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w4-6279f5"},"tenantID":"e2e-worker-w4-6279f5","namespace":"openshift-ingress","gatewayName":"e2e-worker-w4-6279f5"}
{"level":"info","ts":"2026-10-31T18:33:36Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:33:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:33:41Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:33:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:33:51Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:33:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:34:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:34:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:34:59Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:35:28Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:35:30Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:36:16Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:36:19Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:36:21Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w1-da6ea7"},"tenantID":"e2e-worker-w1-da6ea7","namespace":"openshift-ingress","gatewayName":"e2e-worker-w1-da6ea7"}
{"level":"info","ts":"2026-10-31T18:36:38Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:36:40Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:36:43Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:36:45Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:36:48Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w0-a8587d"},"tenantID":"e2e-worker-w0-a8587d","namespace":"openshift-ingress","gatewayName":"e2e-worker-w0-a8587d"}
{"level":"info","ts":"2026-10-31T18:36:50Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:36:53Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w2-88085e"},"tenantID":"e2e-worker-w2-88085e","namespace":"openshift-ingress","gatewayName":"e2e-worker-w2-88085e"}
{"level":"info","ts":"2026-10-31T18:36:58Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"ai-tenant-e2e-worker-w3-3e4a2c"},"tenantID":"e2e-worker-w3-3e4a2c","namespace":"openshift-ingress","gatewayName":"e2e-worker-w3-3e4a2c"}
{"level":"info","ts":"2026-10-31T18:37:10Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:37:12Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"info","ts":"2026-10-31T18:37:39Z","logger":"tenant","msg":"praxis-extproc install applied","maastenantconfig":{"name":"default-tenant","namespace":"models-as-a-service"},"tenantID":"","namespace":"openshift-ingress","gatewayName":"maas-default-gateway"}
{"level":"error","ts":"2026-10-31T18:37:52Z","logger":"leaderelection","msg":"Error retrieving lease lock","lock":"opendatahub/ai-gateway-controller-leader.opendatahub.io","error":"Get \"https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/ai-gateway-controller-leader.opendatahub.io?timeout=5s\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\tk8s.io/client-go@v0.37.0/tools/leaderelection/leaderelection.go:473\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1.1\n\tk8s.io/client-go@v0.37.0/tools/leaderelection/leaderelection.go:292\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/poll.go:48\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1\n\tk8s.io/client-go@v0.37.0/tools/leaderelection/leaderelection.go:285\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\tk8s.io/apimachinery@v0.37.0/pkg/util/wait/backoff.go:172\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew\n\tk8s.io/client-go@v0.37.0/tools/leaderelection/leaderelection.go:284\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\tk8s.io/client-go@v0.37.0/tools/leaderelection/leaderelection.go:221\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\tsigs.k8s.io/controller-runtime@v0.25.2/pkg/manager/internal.go:474"}
{"level":"info","ts":"2026-10-31T18:37:52Z","logger":"leaderelection","msg":"Failed to renew lease","lock":"opendatahub/ai-gateway-controller-leader.opendatahub.io","err":"context deadline exceeded"}
{"level":"error","ts":"2026-10-31T18:37:52Z","logger":"setup","msg":"problem running manager","error":"leader election lost","stacktrace":"main.main\n\tgithub.com/opendatahub-io/ai-gateway-controller/cmd/manager/main.go:223\nruntime.main\n\truntime/proc.go:290"}