+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-4.ec2.internal ++ [[ -n ip-10-0-2-4.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-4.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 10:43:17.446637258 - disable conntrack on geneve port' I0327 10:43:17.446637258 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 10:43:17.464139110 - starting ovnkube-node' I0327 10:43:17.464139110 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-4.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-4.ec2.internal --init-node ip-10-0-2-4.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-4.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 10:43:17.501956 3868 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 10:43:17.501999 3868 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-e04d4b4026.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 10:43:17.503512 3868 kube.go:419] Waiting for certificate I0327 10:43:17.503555 3868 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 10:43:17.503591 3868 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 10:43:17.504750 3868 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 10:43:17.533422 3868 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 10:43:17.533441 3868 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 10:43:17.536555 3868 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 10:43:17.539472 3868 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-dvqlz" I0327 10:43:17.549060 3868 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-dvqlz" I0327 10:43:17.549155 3868 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 10:43:18.504493 3868 kube.go:426] Certificate found I0327 10:43:18.505173 3868 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 10:43:18.506346 3868 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 10:43:18.552042 3868 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 10:38:17 +0000 UTC" deadline="2026-03-28 06:16:53.966203332 +0000 UTC" I0327 10:43:18.552171 3868 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h33m35.41404668s" I0327 10:43:18.572981 3868 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 10:43:18.590078 3868 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 10:43:18.590460 3868 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 10:43:18.590521 3868 factory.go:531] Starting watch factory I0327 10:43:18.590604 3868 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590610 3868 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590623 3868 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590629 3868 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590650 3868 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590728 3868 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590746 3868 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590761 3868 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590880 3868 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.590906 3868 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.591388 3868 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.591484 3868 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.595458 3868 controller_manager.go:368] Starting the ovnkube controller I0327 10:43:18.595467 3868 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-4.ec2.internal I0327 10:43:18.595509 3868 controller_manager.go:393] NBDB zone sync took: 35.162µs I0327 10:43:18.595517 3868 factory.go:531] Starting watch factory I0327 10:43:18.608030 3868 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.609272 3868 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.610324 3868 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.610739 3868 ovn_northd.go:121] Registering OVN northd metrics I0327 10:43:18.610832 3868 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.610992 3868 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 10:43:18.615298 3868 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 10:43:18.615322 3868 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 10:43:18.616450 3868 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.618600 3868 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 10:43:18.632009 3868 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 10:43:18.697176 3868 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 10:43:18.697194 3868 factory.go:1890] *v1.Node informer cache synced successfully I0327 10:43:18.697197 3868 factory.go:1890] *v1.Service informer cache synced successfully I0327 10:43:18.697201 3868 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 10:43:18.697204 3868 factory.go:1890] *v1.Pod informer cache synced successfully I0327 10:43:18.697206 3868 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 10:43:18.697275 3868 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 10:43:18.697273 3868 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 10:43:18.697294 3868 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 10:43:18.697298 3868 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 10:43:18.700335 3868 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 10:43:18.700746 3868 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 10:43:18.701049 3868 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 10:43:18.701059 3868 factory.go:1890] *v1.Node informer cache synced successfully I0327 10:43:18.701063 3868 factory.go:1890] *v1.Service informer cache synced successfully I0327 10:43:18.701066 3868 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 10:43:18.701069 3868 factory.go:1890] *v1.Pod informer cache synced successfully I0327 10:43:18.701072 3868 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 10:43:18.702790 3868 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 10:43:18.702797 3868 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 10:43:18.702842 3868 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.702855 3868 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.705084 3868 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.707056 3868 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 10:43:18.707067 3868 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 10:43:18.708149 3868 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 10:43:18.708203 3868 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.708223 3868 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.710368 3868 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.712308 3868 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 10:43:18.713456 3868 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 10:43:18.713507 3868 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.713518 3868 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.715753 3868 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.717775 3868 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 10:43:18.718837 3868 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 10:43:18.718895 3868 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.718907 3868 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.720967 3868 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.722935 3868 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 10:43:18.724080 3868 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 10:43:18.724126 3868 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.724137 3868 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.726291 3868 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.728177 3868 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 10:43:18.729259 3868 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 10:43:18.729418 3868 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 10:43:18.729433 3868 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 10:43:18.731598 3868 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 10:43:18.733514 3868 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 10:43:18.734598 3868 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 10:43:18.734731 3868 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.734750 3868 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.734771 3868 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.734757 3868 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.736992 3868 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.737127 3868 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 10:43:18.738979 3868 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 10:43:18.740356 3868 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 10:43:18.740365 3868 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 10:43:18.740372 3868 factory.go:636] Watch Factory start up complete, took: 149.853679ms I0327 10:43:18.740435 3868 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 10:43:18.744097 3868 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 10:43:18.744108 3868 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 10:43:18.744115 3868 factory.go:636] Watch Factory start up complete, took: 148.596289ms I0327 10:43:18.744122 3868 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-4.ec2.internal" zone I0327 10:43:18.779864 3868 default_node_network_controller.go:709] Initializing the default node network controller I0327 10:43:18.838091 3868 udn_isolation.go:101] Starting UDN host isolation manager I0327 10:43:18.844583 3868 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 10:43:18.884347 3868 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 10:43:18.884419 3868 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 10:43:18.884437 3868 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 10:43:18.915451 3868 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 10:43:18.915505 3868 default_node_network_controller.go:808] Node ip-10-0-2-4.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0327 10:43:18.920313 3868 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.4"] k8s.ovn.org/zone-name:ip-10-0-2-4.ec2.internal] on node ip-10-0-2-4.ec2.internal I0327 10:43:18.940636 3868 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 10:43:18.946566 3868 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 10:43:18.946746 3868 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0327 10:43:18.946906 3868 gateway_init.go:263] Preparing Gateway I0327 10:43:18.946917 3868 gateway_shared_intf.go:1658] Creating new gateway I0327 10:43:18.994189 3868 controller_manager.go:427] Waiting for node in zone sync took: 250.055567ms I0327 10:43:18.999143 3868 controller_manager.go:608] Switching to transit router for layer2 networks I0327 10:43:18.999169 3868 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-4.ec2.internal I0327 10:43:19.013422 3868 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 10:43:19.019517 3868 controller_manager.go:288] SCTP support detected in OVN I0327 10:43:19.020169 3868 services_controller.go:69] Creating services controller for network=default I0327 10:43:19.020248 3868 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 10:43:19.020336 3868 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 10:43:19.020375 3868 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 10:43:19.020394 3868 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 10:43:19.020406 3868 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 10:43:19.020622 3868 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 10:43:19.020672 3868 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 10:43:19.020734 3868 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 50.345µs I0327 10:43:19.020748 3868 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 76.428µs I0327 10:43:19.020755 3868 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 10:43:19.020761 3868 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 10:43:19.020767 3868 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 106.175µs I0327 10:43:19.020777 3868 udn_enabled_svc.go:95] Performing full resync I0327 10:43:19.020772 3868 default_network_controller.go:321] Starting the default network controller I0327 10:43:19.020793 3868 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 13.368µs I0327 10:43:19.021143 3868 udn_enabled_svc.go:99] Waiting for handler to sync I0327 10:43:19.021157 3868 udn_enabled_svc.go:104] Starting worker I0327 10:43:19.026100 3868 iptables.go:108] Creating table: filter chain: FORWARD I0327 10:43:19.029619 3868 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 10:43:19.029675 3868 repair.go:33] Syncing exgw routes took 43.577µs I0327 10:43:19.029694 3868 default_network_controller.go:407] Starting all the Watchers... I0327 10:43:19.029873 3868 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 10:43:19.029889 3868 namespace.go:98] [openshift-logging] adding namespace I0327 10:43:19.029902 3868 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 10:43:19.029901 3868 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 10:43:19.029913 3868 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 10:43:19.029917 3868 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 10:43:19.029931 3868 namespace.go:98] [openshift-authentication] adding namespace I0327 10:43:19.029931 3868 namespace.go:98] [kube-node-lease] adding namespace I0327 10:43:19.029934 3868 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 10:43:19.029938 3868 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 10:43:19.029940 3868 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 10:43:19.029935 3868 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 10:43:19.029947 3868 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 10:43:19.029973 3868 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 10:43:19.029873 3868 namespace.go:98] [dedicated-admin] adding namespace I0327 10:43:19.030996 3868 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 1.110661ms I0327 10:43:19.031012 3868 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 10:43:19.031732 3868 namespace.go:102] [openshift-operators-redhat] adding namespace took 711.637µs I0327 10:43:19.031796 3868 namespace.go:98] [kube-public] adding namespace I0327 10:43:19.032536 3868 namespace.go:102] [openshift-logging] adding namespace took 2.635713ms I0327 10:43:19.032556 3868 namespace.go:98] [openshift-multus] adding namespace I0327 10:43:19.033133 3868 namespace.go:102] [openshift-customer-monitoring] adding namespace took 3.225251ms I0327 10:43:19.033152 3868 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 10:43:19.033789 3868 namespace.go:102] [openshift-backplane-csm] adding namespace took 3.873359ms I0327 10:43:19.033808 3868 namespace.go:98] [openshift] adding namespace I0327 10:43:19.033953 3868 iptables.go:108] Creating table: filter chain: OUTPUT I0327 10:43:19.034345 3868 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 4.424504ms I0327 10:43:19.034362 3868 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 10:43:19.035131 3868 namespace.go:102] [openshift-authentication-operator] adding namespace took 5.200396ms I0327 10:43:19.035151 3868 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 10:43:19.035581 3868 namespace.go:102] [openshift-authentication] adding namespace took 5.639336ms I0327 10:43:19.035603 3868 namespace.go:98] [openshift-etcd] adding namespace I0327 10:43:19.036208 3868 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.259184ms I0327 10:43:19.036229 3868 namespace.go:98] [openshift-marketplace] adding namespace I0327 10:43:19.036800 3868 namespace.go:102] [kube-node-lease] adding namespace took 6.85687ms I0327 10:43:19.036817 3868 namespace.go:98] [openshift-network-operator] adding namespace I0327 10:43:19.037411 3868 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.460162ms I0327 10:43:19.037432 3868 namespace.go:98] [openshift-insights] adding namespace I0327 10:43:19.037951 3868 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.996914ms I0327 10:43:19.037966 3868 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 10:43:19.038348 3868 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.390527ms I0327 10:43:19.038366 3868 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 10:43:19.038847 3868 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.887903ms I0327 10:43:19.038874 3868 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 10:43:19.039322 3868 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 9.338291ms I0327 10:43:19.039336 3868 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 10:43:19.039728 3868 namespace.go:102] [dedicated-admin] adding namespace took 9.703739ms I0327 10:43:19.039742 3868 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace W0327 10:43:19.040231 3868 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-85prw: pod openshift-multus/network-metrics-daemon-85prw: no pod IPs found W0327 10:43:19.040242 3868 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-kfv96: pod openshift-multus/network-metrics-daemon-kfv96: no pod IPs found I0327 10:43:19.040275 3868 namespace.go:102] [kube-public] adding namespace took 8.465868ms I0327 10:43:19.040293 3868 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 10:43:19.040681 3868 namespace.go:102] [openshift-multus] adding namespace took 8.119038ms I0327 10:43:19.040694 3868 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 10:43:19.041078 3868 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.918809ms I0327 10:43:19.041105 3868 namespace.go:98] [openshift-config-managed] adding namespace I0327 10:43:19.041080 3868 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 10:43:19.041121 3868 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-4.ec2.internal","mac-address":"0e:7a:31:ea:e0:09","ip-addresses":["10.0.2.4/24"],"ip-address":"10.0.2.4/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:79b4a826-847f-402e-ac89-2015999bfb74 k8s.ovn.org/node-encap-ips:["10.0.2.4"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.4/24"} k8s.ovn.org/zone-name:ip-10-0-2-4.ec2.internal] on node ip-10-0-2-4.ec2.internal I0327 10:43:19.041527 3868 namespace.go:102] [openshift] adding namespace took 7.713851ms I0327 10:43:19.041540 3868 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 10:43:19.041994 3868 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.625995ms I0327 10:43:19.042009 3868 namespace.go:98] [default] adding namespace I0327 10:43:19.042366 3868 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.209763ms I0327 10:43:19.042379 3868 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 10:43:19.042735 3868 namespace.go:102] [openshift-etcd] adding namespace took 7.127171ms I0327 10:43:19.042748 3868 namespace.go:98] [openshift-package-operator] adding namespace I0327 10:43:19.043241 3868 namespace.go:102] [openshift-marketplace] adding namespace took 7.005775ms I0327 10:43:19.043255 3868 namespace.go:98] [openshift-operators] adding namespace I0327 10:43:19.043628 3868 namespace.go:102] [openshift-network-operator] adding namespace took 6.806845ms I0327 10:43:19.043652 3868 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 10:43:19.044072 3868 namespace.go:102] [openshift-insights] adding namespace took 6.63213ms I0327 10:43:19.044091 3868 namespace.go:98] [openshift-node] adding namespace I0327 10:43:19.044494 3868 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.523495ms I0327 10:43:19.044506 3868 namespace.go:98] [openshift-config-operator] adding namespace I0327 10:43:19.044969 3868 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.59795ms I0327 10:43:19.044983 3868 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 10:43:19.045377 3868 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.498429ms I0327 10:43:19.045389 3868 namespace.go:98] [openshift-apiserver] adding namespace I0327 10:43:19.045865 3868 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.524115ms I0327 10:43:19.045876 3868 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 10:43:19.046350 3868 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.602867ms I0327 10:43:19.046364 3868 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 10:43:19.046737 3868 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.439837ms I0327 10:43:19.046748 3868 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 10:43:19.047167 3868 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.467957ms I0327 10:43:19.047180 3868 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 10:43:19.047523 3868 namespace.go:102] [openshift-config-managed] adding namespace took 6.409785ms I0327 10:43:19.047533 3868 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0327 10:43:19.047995 3868 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.449593ms I0327 10:43:19.048009 3868 namespace.go:98] [open-cluster-management-2pa0ron8lulpu1e8444fbruk00uofggt] adding namespace I0327 10:43:19.048373 3868 namespace.go:102] [default] adding namespace took 6.358493ms I0327 10:43:19.048383 3868 namespace.go:98] [openshift-dns] adding namespace I0327 10:43:19.048789 3868 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.406938ms I0327 10:43:19.048798 3868 namespace.go:98] [openshift-monitoring] adding namespace I0327 10:43:19.049177 3868 namespace.go:102] [openshift-package-operator] adding namespace took 6.424304ms I0327 10:43:19.049190 3868 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 10:43:19.049567 3868 namespace.go:102] [openshift-operators] adding namespace took 6.308116ms I0327 10:43:19.049579 3868 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 10:43:19.049955 3868 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.296809ms I0327 10:43:19.049976 3868 namespace.go:98] [openshift-network-console] adding namespace I0327 10:43:19.050438 3868 namespace.go:102] [openshift-node] adding namespace took 6.342178ms I0327 10:43:19.050452 3868 namespace.go:98] [openshift-infra] adding namespace I0327 10:43:19.050953 3868 namespace.go:102] [openshift-config-operator] adding namespace took 6.44173ms I0327 10:43:19.050968 3868 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 10:43:19.051329 3868 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.341657ms I0327 10:43:19.051340 3868 namespace.go:98] [openshift-host-network] adding namespace I0327 10:43:19.051737 3868 namespace.go:102] [openshift-apiserver] adding namespace took 6.343914ms I0327 10:43:19.051767 3868 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 10:43:19.052186 3868 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.304068ms I0327 10:43:19.052199 3868 namespace.go:98] [openshift-console] adding namespace I0327 10:43:19.052556 3868 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.186336ms I0327 10:43:19.052569 3868 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 10:43:19.052987 3868 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.234744ms I0327 10:43:19.052998 3868 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 10:43:19.053366 3868 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.181316ms I0327 10:43:19.053378 3868 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 10:43:19.053736 3868 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.197908ms I0327 10:43:19.053749 3868 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 10:43:19.054188 3868 namespace.go:102] [open-cluster-management-2pa0ron8lulpu1e8444fbruk00uofggt] adding namespace took 6.174179ms I0327 10:43:19.054201 3868 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 10:43:19.055114 3868 namespace.go:102] [openshift-dns] adding namespace took 6.724038ms I0327 10:43:19.055132 3868 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 10:43:19.055673 3868 namespace.go:102] [openshift-monitoring] adding namespace took 6.867647ms I0327 10:43:19.055689 3868 namespace.go:98] [openshift-cluster-version] adding namespace I0327 10:43:19.056254 3868 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.058586ms I0327 10:43:19.056267 3868 namespace.go:98] [openshift-machine-api] adding namespace I0327 10:43:19.056745 3868 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.16089ms I0327 10:43:19.056760 3868 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 10:43:19.057241 3868 namespace.go:102] [openshift-network-console] adding namespace took 7.259607ms I0327 10:43:19.057251 3868 namespace.go:98] [openshift-image-registry] adding namespace I0327 10:43:19.057688 3868 namespace.go:102] [openshift-infra] adding namespace took 7.229592ms I0327 10:43:19.058218 3868 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.244613ms I0327 10:43:19.058232 3868 namespace.go:98] [openshift-dns-operator] adding namespace I0327 10:43:19.058514 3868 default_node_network_controller.go:518] Node connection status = connected I0327 10:43:19.058850 3868 namespace.go:102] [openshift-host-network] adding namespace took 7.503627ms I0327 10:43:19.058870 3868 namespace.go:98] [openshift-console-operator] adding namespace I0327 10:43:19.059309 3868 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.533542ms I0327 10:43:19.059322 3868 namespace.go:98] [kube-system] adding namespace W0327 10:43:19.059774 3868 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-bh89c: pod openshift-network-diagnostics/network-check-target-bh89c: no pod IPs found W0327 10:43:19.059788 3868 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-slcss: pod openshift-network-diagnostics/network-check-target-slcss: no pod IPs found I0327 10:43:19.059825 3868 namespace.go:102] [openshift-console] adding namespace took 7.618796ms I0327 10:43:19.060310 3868 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.736463ms I0327 10:43:19.060322 3868 namespace.go:98] [openshift-backplane] adding namespace I0327 10:43:19.060702 3868 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.699132ms I0327 10:43:19.061118 3868 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.73537ms I0327 10:43:19.061493 3868 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.739103ms I0327 10:43:19.062027 3868 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.816285ms I0327 10:43:19.062671 3868 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.531776ms I0327 10:43:19.062690 3868 namespace.go:98] [openshift-config] adding namespace I0327 10:43:19.063167 3868 namespace.go:102] [openshift-cluster-version] adding namespace took 7.471579ms I0327 10:43:19.063184 3868 namespace.go:98] [openshift-controller-manager] adding namespace I0327 10:43:19.063592 3868 namespace.go:102] [openshift-machine-api] adding namespace took 7.3189ms I0327 10:43:19.063610 3868 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 10:43:19.064052 3868 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.286107ms I0327 10:43:19.064070 3868 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 10:43:19.064486 3868 namespace.go:102] [openshift-image-registry] adding namespace took 7.229439ms I0327 10:43:19.064984 3868 namespace.go:102] [openshift-dns-operator] adding namespace took 6.746375ms W0327 10:43:19.065377 3868 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-4vkbk: pod kube-system/global-pull-secret-syncer-4vkbk: no pod IPs found I0327 10:43:19.065428 3868 namespace.go:102] [openshift-console-operator] adding namespace took 6.550385ms I0327 10:43:19.065441 3868 namespace.go:98] [openshift-ingress] adding namespace I0327 10:43:19.065885 3868 namespace.go:102] [kube-system] adding namespace took 6.557371ms I0327 10:43:19.066288 3868 namespace.go:102] [openshift-backplane] adding namespace took 5.961084ms I0327 10:43:19.066669 3868 namespace.go:102] [openshift-config] adding namespace took 3.973495ms I0327 10:43:19.067078 3868 namespace.go:102] [openshift-controller-manager] adding namespace took 3.886731ms I0327 10:43:19.067439 3868 namespace.go:102] [openshift-ingress-operator] adding namespace took 3.823026ms I0327 10:43:19.067953 3868 namespace.go:102] [openshift-console-user-settings] adding namespace took 3.877431ms I0327 10:43:19.068351 3868 namespace.go:102] [openshift-ingress] adding namespace took 2.905367ms I0327 10:43:19.068746 3868 zone_ic_handler.go:186] Time taken to create transit switch: 298.509µs I0327 10:43:19.068845 3868 master.go:532] Adding or Updating local node "ip-10-0-2-4.ec2.internal" for network "default" I0327 10:43:19.068865 3868 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-188.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-188.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-188.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-188.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-188.ec2.internal W0327 10:43:19.076220 3868 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-85prw: pod openshift-multus/network-metrics-daemon-85prw: no pod IPs found W0327 10:43:19.076238 3868 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-bh89c: pod openshift-network-diagnostics/network-check-target-bh89c: no pod IPs found W0327 10:43:19.076244 3868 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-4vkbk: pod kube-system/global-pull-secret-syncer-4vkbk: no pod IPs found I0327 10:43:19.077232 3868 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-4.ec2.internal for the network default I0327 10:43:19.078050 3868 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 10:43:19.078087 3868 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 10:43:19.078102 3868 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 10:43:19.078118 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-188.ec2.internal in network "default" I0327 10:43:19.078155 3868 node_tracker.go:235] Node ip-10-0-1-188.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-188.ec2.internal" I0327 10:43:19.078169 3868 node_tracker.go:169] Node ip-10-0-1-188.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:19.078179 3868 services_controller.go:574] Full service sync requested for network=default I0327 10:43:19.078191 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-4.ec2.internal in network "default" I0327 10:43:19.078224 3868 node_tracker.go:169] Node ip-10-0-2-4.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:19.078232 3868 services_controller.go:574] Full service sync requested for network=default I0327 10:43:19.178706 3868 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 10:43:19.178744 3868 services_controller.go:208] Setting up event handlers for services for network=default I0327 10:43:19.178821 3868 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 10:43:19.178945 3868 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 10:43:19.178968 3868 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 10:43:19.279509 3868 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 10:43:19.279529 3868 repair.go:60] Starting repairing loop for services I0327 10:43:19.279611 3868 repair.go:131] Deleted 0 stale service LBs I0327 10:43:19.279638 3868 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 10:43:19.279656 3868 repair.go:62] Finished repairing loop for services: 129.55µs I0327 10:43:19.279681 3868 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 10:43:19.279691 3868 services_controller.go:254] Starting workers for network=default I0327 10:43:19.279988 3868 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-slcss, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-slcss: suppressed error logged: pod openshift-network-diagnostics/network-check-target-slcss: no pod IPs found I0327 10:43:19.280023 3868 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-kfv96, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-kfv96: suppressed error logged: pod openshift-multus/network-metrics-daemon-kfv96: no pod IPs found I0327 10:43:19.280030 3868 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-bh89c] creating logical port openshift-network-diagnostics_network-check-target-bh89c for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:19.280258 3868 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-bh89c I0327 10:43:19.280287 3868 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-4vkbk] creating logical port kube-system_global-pull-secret-syncer-4vkbk for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:19.280260 3868 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-85prw] creating logical port openshift-multus_network-metrics-daemon-85prw for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:19.280367 3868 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-4vkbk I0327 10:43:19.280415 3868 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-85prw I0327 10:43:19.302550 3868 pod.go:62] [kube-system/global-pull-secret-syncer-4vkbk] pod update took 22.205572ms I0327 10:43:19.302569 3868 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-4vkbk] addLogicalPort annotation time took 22.224971ms I0327 10:43:19.303443 3868 pods.go:271] [kube-system/global-pull-secret-syncer-4vkbk] addLogicalPort took 23.158253ms, libovsdb time 654.008µs I0327 10:43:19.304456 3868 pod.go:62] [openshift-multus/network-metrics-daemon-85prw] pod update took 24.062243ms I0327 10:43:19.304473 3868 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-85prw] addLogicalPort annotation time took 24.079521ms I0327 10:43:19.305290 3868 pod.go:62] [openshift-network-diagnostics/network-check-target-bh89c] pod update took 25.149351ms I0327 10:43:19.305311 3868 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-bh89c] addLogicalPort annotation time took 25.17197ms I0327 10:43:19.305336 3868 pods.go:271] [openshift-multus/network-metrics-daemon-85prw] addLogicalPort took 25.085492ms, libovsdb time 676.762µs I0327 10:43:19.306166 3868 pods.go:271] [openshift-network-diagnostics/network-check-target-bh89c] addLogicalPort took 26.159981ms, libovsdb time 597.489µs I0327 10:43:19.306553 3868 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 10:43:19.306577 3868 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 10:43:19.306590 3868 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 10:43:19.306595 3868 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 10:43:19.306797 3868 repair.go:29] Repairing admin network policies took 197.459µs I0327 10:43:19.306905 3868 repair.go:92] Repairing baseline admin network policies took 98.453µs I0327 10:43:19.307294 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 10:43:19.307315 3868 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 10:43:19.308984 3868 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 10:43:19.309003 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 10:43:19.309011 3868 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 10:43:19.309505 3868 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 10:43:19.309521 3868 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 10:43:19.309531 3868 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 10:43:19.311804 3868 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 10:43:19.311823 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 10:43:19.311831 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 10:43:19.313173 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 10:43:19.313190 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 10:43:19.313198 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 10:43:19.313637 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 10:43:19.313656 3868 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 10:43:19.313665 3868 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 10:43:19.315519 3868 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 10:43:19.315536 3868 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 10:43:19.315547 3868 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 10:43:19.316556 3868 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 10:43:19.316572 3868 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 10:43:19.316581 3868 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 10:43:19.317659 3868 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 10:43:19.317682 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 10:43:19.317691 3868 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 10:43:19.318006 3868 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 10:43:19.318024 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 10:43:19.318032 3868 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 10:43:19.318535 3868 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 10:43:19.318553 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 10:43:19.318562 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 10:43:19.320061 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 10:43:19.320080 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 10:43:19.320086 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 10:43:19.320368 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 10:43:19.320386 3868 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 10:43:19.320391 3868 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 10:43:19.321665 3868 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 10:43:19.321683 3868 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 10:43:19.321695 3868 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 10:43:19.322953 3868 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 10:43:19.322972 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 10:43:19.323389 3868 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 10:43:19.323401 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 10:43:19.324135 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 10:43:19.324153 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 10:43:19.324161 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 10:43:19.324690 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 10:43:19.324725 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 10:43:19.324739 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 10:43:19.325244 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 10:43:19.325262 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 10:43:19.325285 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 10:43:19.325924 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 10:43:19.325939 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 10:43:19.325945 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 10:43:19.327638 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 10:43:19.327654 3868 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 10:43:19.327663 3868 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 10:43:19.328485 3868 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 10:43:19.328505 3868 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 10:43:19.328511 3868 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 10:43:19.328769 3868 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 10:43:19.328784 3868 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 10:43:19.328793 3868 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 10:43:19.329799 3868 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 10:43:19.329816 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 10:43:19.329823 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 10:43:19.330282 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 10:43:19.330299 3868 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 10:43:19.330306 3868 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 10:43:19.330793 3868 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 10:43:19.330810 3868 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 10:43:19.330819 3868 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 10:43:19.332423 3868 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 10:43:19.332442 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 10:43:19.332450 3868 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 10:43:19.332946 3868 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 10:43:19.332963 3868 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 10:43:19.332972 3868 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 10:43:19.333477 3868 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 10:43:19.333494 3868 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 10:43:19.333501 3868 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 10:43:19.334008 3868 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 10:43:19.334025 3868 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 10:43:19.334035 3868 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 10:43:19.334842 3868 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 10:43:19.338839 3868 egressfirewall.go:319] Starting EgressFirewall controller I0327 10:43:19.338869 3868 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 10:43:19.338892 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 10:43:19.338904 3868 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 10:43:19.338930 3868 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 10:43:19.338953 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 10:43:19.338964 3868 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 10:43:19.339211 3868 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 10:43:19.339224 3868 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 10:43:19.339233 3868 egressqos.go:193] Setting up event handlers for EgressQoS I0327 10:43:19.339306 3868 egressfirewall.go:628] Syncing node "ip-10-0-1-188.ec2.internal" for egress firewall I0327 10:43:19.339318 3868 egressfirewall.go:628] Syncing node "ip-10-0-2-4.ec2.internal" for egress firewall I0327 10:43:19.339346 3868 egressqos.go:245] Starting EgressQoS Controller I0327 10:43:19.339355 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 10:43:19.339361 3868 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 10:43:19.339366 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 10:43:19.339371 3868 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 10:43:19.339376 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 10:43:19.339380 3868 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 10:43:19.339384 3868 egressqos.go:259] Repairing EgressQoSes I0327 10:43:19.339387 3868 egressqos.go:399] Starting repairing loop for egressqos I0327 10:43:19.339453 3868 egressqos.go:401] Finished repairing loop for egressqos: 64.317µs I0327 10:43:19.339471 3868 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 10:43:19.339492 3868 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-4.ec2.internal I0327 10:43:19.339532 3868 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-4.ec2.internal : 40.394µs I0327 10:43:19.339584 3868 egressservice_zone.go:210] Starting Egress Services Controller I0327 10:43:19.339598 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 10:43:19.339606 3868 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 10:43:19.339611 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 10:43:19.339616 3868 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 10:43:19.339621 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 10:43:19.339625 3868 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 10:43:19.339638 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 10:43:19.339643 3868 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 10:43:19.339647 3868 egressservice_zone.go:228] Repairing Egress Services I0327 10:43:19.339918 3868 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 10:43:19.339927 3868 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 10:43:19.339940 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-188.ec2.internal I0327 10:43:19.339949 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-188.ec2.internal: 10.87µs I0327 10:43:19.339955 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-4.ec2.internal I0327 10:43:19.339959 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-4.ec2.internal: 4.16µs I0327 10:43:19.340025 3868 default_network_controller.go:568] Completing all the Watchers took 310.321937ms I0327 10:43:19.340037 3868 default_network_controller.go:572] Starting unidling controllers I0327 10:43:19.340048 3868 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 10:43:19.340053 3868 unidle.go:63] Populating Initial ContollerEvent events I0327 10:43:19.340106 3868 unidle.go:79] Setting up event handlers for services I0327 10:43:19.340153 3868 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 10:43:19.559210 3868 default_node_network_controller.go:518] Node connection status = connected I0327 10:43:20.059101 3868 default_node_network_controller.go:518] Node connection status = connected I0327 10:43:20.340911 3868 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 10:43:20.559372 3868 default_node_network_controller.go:518] Node connection status = connected I0327 10:43:20.575045 3868 bridgeconfig.go:537] Gateway is ready I0327 10:43:20.575064 3868 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 10:43:20.585687 3868 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.4. Updating OVN encap IP. I0327 10:43:20.590228 3868 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.4 - it is already configured I0327 10:43:20.590241 3868 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.4/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 10:43:20.590470 3868 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.4/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-4.ec2.internal","mac-address":"0e:7a:31:ea:e0:09","ip-addresses":["10.0.2.4/24"],"ip-address":"10.0.2.4/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:79b4a826-847f-402e-ac89-2015999bfb74 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.4/24"}] on node ip-10-0-2-4.ec2.internal I0327 10:43:20.611949 3868 egressfirewall.go:628] Syncing node "ip-10-0-2-4.ec2.internal" for egress firewall I0327 10:43:20.611970 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-4.ec2.internal I0327 10:43:20.611974 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-4.ec2.internal in network "default" I0327 10:43:20.611982 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-4.ec2.internal: 17.48µs I0327 10:43:20.612035 3868 node_tracker.go:169] Node ip-10-0-2-4.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:20.612051 3868 services_controller.go:574] Full service sync requested for network=default I0327 10:43:20.612099 3868 master.go:532] Adding or Updating local node "ip-10-0-2-4.ec2.internal" for network "default" I0327 10:43:20.612842 3868 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.4 I0327 10:43:20.612960 3868 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-4.ec2.internal I0327 10:43:20.616844 3868 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-4.ec2.internal. Updating no re-route policies I0327 10:43:20.616920 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-wjnkl I0327 10:43:20.616937 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-wjnkl I0327 10:43:20.616963 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-wjnkl after 0 failed attempt(s) I0327 10:43:20.616978 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-bh89c I0327 10:43:20.616986 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-bh89c I0327 10:43:20.616989 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-85prw I0327 10:43:20.617003 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-85prw I0327 10:43:20.617010 3868 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-bh89c] creating logical port openshift-network-diagnostics_network-check-target-bh89c for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:20.617016 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-kblqf I0327 10:43:20.617026 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-kblqf I0327 10:43:20.617026 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-zg84h I0327 10:43:20.617026 3868 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-5zg6j I0327 10:43:20.617040 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-zg84h I0327 10:43:20.617042 3868 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-5zg6j I0327 10:43:20.617047 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fqsbb I0327 10:43:20.617050 3868 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-4.ec2.internal I0327 10:43:20.617055 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-2kzqm I0327 10:43:20.616989 3868 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-nxm5x I0327 10:43:20.617063 3868 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-4.ec2.internal I0327 10:43:20.617061 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-2kzqm I0327 10:43:20.617071 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-4fj4n I0327 10:43:20.617080 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-4fj4n I0327 10:43:20.617084 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-2kzqm after 0 failed attempt(s) I0327 10:43:20.617083 3868 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-nxm5x I0327 10:43:20.617031 3868 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-85prw] creating logical port openshift-multus_network-metrics-daemon-85prw for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:20.617109 3868 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-4.ec2.internal after 0 failed attempt(s) I0327 10:43:20.617015 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-4.ec2.internal I0327 10:43:20.617118 3868 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-nxm5x after 0 failed attempt(s) I0327 10:43:20.617131 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-4.ec2.internal I0327 10:43:20.617047 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-slcss I0327 10:43:20.617149 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-slcss I0327 10:43:20.617074 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-2cwtw I0327 10:43:20.617148 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-kblqf after 0 failed attempt(s) I0327 10:43:20.617158 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-4.ec2.internal after 0 failed attempt(s) I0327 10:43:20.617166 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-2cwtw I0327 10:43:20.617010 3868 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-4vkbk I0327 10:43:20.617174 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-4fj4n after 0 failed attempt(s) I0327 10:43:20.617063 3868 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-5zg6j after 0 failed attempt(s) I0327 10:43:20.617184 3868 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-4vkbk I0327 10:43:20.617190 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-2cwtw after 0 failed attempt(s) I0327 10:43:20.617001 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-kfv96 I0327 10:43:20.617208 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-kfv96 I0327 10:43:20.617224 3868 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-4vkbk] creating logical port kube-system_global-pull-secret-syncer-4vkbk for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:20.617039 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-sns6j I0327 10:43:20.617245 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-zg84h after 0 failed attempt(s) I0327 10:43:20.617246 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-sns6j I0327 10:43:20.617265 3868 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-kfv96, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-kfv96: suppressed error logged: pod openshift-multus/network-metrics-daemon-kfv96: no pod IPs found I0327 10:43:20.617203 3868 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-slcss, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-slcss: suppressed error logged: pod openshift-network-diagnostics/network-check-target-slcss: no pod IPs found I0327 10:43:20.617275 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-sns6j after 0 failed attempt(s) I0327 10:43:20.617059 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-fqsbb I0327 10:43:20.617294 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-fqsbb after 0 failed attempt(s) I0327 10:43:20.617951 3868 pods.go:271] [openshift-network-diagnostics/network-check-target-bh89c] addLogicalPort took 949.822µs, libovsdb time 340.276µs I0327 10:43:20.617961 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-bh89c after 0 failed attempt(s) I0327 10:43:20.618383 3868 pods.go:271] [openshift-multus/network-metrics-daemon-85prw] addLogicalPort took 1.355414ms, libovsdb time 682.387µs I0327 10:43:20.618401 3868 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-85prw after 0 failed attempt(s) I0327 10:43:20.618477 3868 pods.go:271] [kube-system/global-pull-secret-syncer-4vkbk] addLogicalPort took 1.26608ms, libovsdb time 754.89µs I0327 10:43:20.618486 3868 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-4vkbk after 0 failed attempt(s) I0327 10:43:20.630643 3868 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 10:43:20.641784 3868 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 10:43:20.645539 3868 iptables.go:108] Creating table: nat chain: OUTPUT I0327 10:43:20.671702 3868 iptables.go:108] Creating table: nat chain: PREROUTING I0327 10:43:20.752423 3868 iptables.go:108] Creating table: filter chain: FORWARD I0327 10:43:20.769790 3868 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 10:43:20.769874 3868 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 10:43:20.769923 3868 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 10:43:20.770010 3868 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 10:43:20.770058 3868 gateway_init.go:325] Gateway and management port readiness took 1.714757472s I0327 10:43:20.770084 3868 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 10:43:20.770127 3868 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 10:43:20.770140 3868 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 10:43:20.770149 3868 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 10:43:20.770202 3868 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 10:43:20.770280 3868 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 50.33µs I0327 10:43:20.770483 3868 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 10:43:20.770508 3868 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 17.342µs I0327 10:43:20.770516 3868 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.752µs I0327 10:43:20.770521 3868 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 10:43:20.770529 3868 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 10:43:20.770534 3868 default_node_network_controller.go:898] Starting the default node network controller I0327 10:43:20.770538 3868 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 10:43:20.770634 3868 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 6.653µs I0327 10:43:20.770665 3868 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 10:43:20.770673 3868 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 10:43:20.770695 3868 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a256d0 0xc0002bf9d0 0xc0002bfa40 0xc0002bfab0 0xc0002bfb20 0xc0002bfb90 0xc0002bfc00 0xc000c62550 0xc0002bfc70 0xc0002bfce0 0xc000c626e0 0xc006527180 0xc0002bfd50 map[0x2f67040:0xc002a85f20 0x2f67580:0xc00541a1e0 0x2f68000:0xc002a85e60 0x2f682a0:0xc002a85ec0 0x2f687e0:0xc00541a000 0x2f68a80:0xc00541a060 0x2f68d20:0xc00541a0c0 0x2f69500:0xc00541a240 0x2f697a0:0xc00541a2a0 0x2f93d40:0xc000f53380 0x2f940e0:0xc000f534a0 0x2f94bc0:0xc000c645a0 0x2f96520:0xc001cb6000 0x2f97000:0xc002504a80 0x2f97e80:0xc001507ec0] 0xc0000d37a0 0} I0327 10:43:20.770771 3868 gateway.go:151] Starting gateway service sync I0327 10:43:20.771158 3868 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 10:43:20.773508 3868 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 10:43:20.796155 3868 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 10:43:20.797374 3868 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 10:43:20.798562 3868 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 10:43:20.832354 3868 gateway.go:168] Gateway service sync done. Time taken: 61.574397ms I0327 10:43:20.833277 3868 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30746 I0327 10:43:20.833451 3868 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a256d0 0xc0002bf9d0 0xc0002bfa40 0xc0002bfab0 0xc0002bfb20 0xc0002bfb90 0xc0002bfc00 0xc000c62550 0xc0002bfc70 0xc0002bfce0 0xc000c626e0 0xc006527180 0xc0002bfd50 map[0x2f67040:0xc002a85f20 0x2f67580:0xc00541a1e0 0x2f68000:0xc002a85e60 0x2f682a0:0xc002a85ec0 0x2f687e0:0xc00541a000 0x2f68a80:0xc00541a060 0x2f68d20:0xc00541a0c0 0x2f69500:0xc00541a240 0x2f697a0:0xc00541a2a0 0x2f93d40:0xc000f53380 0x2f940e0:0xc000f534a0 0x2f94bc0:0xc000c645a0 0x2f96520:0xc001cb6000 0x2f97000:0xc002504a80 0x2f97e80:0xc001507ec0] 0xc0000d37a0 0} I0327 10:43:20.833851 3868 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a256d0 0xc0002bf9d0 0xc0002bfa40 0xc0002bfab0 0xc0002bfb20 0xc0002bfb90 0xc0002bfc00 0xc000c62550 0xc0002bfc70 0xc0002bfce0 0xc000c626e0 0xc006527180 0xc0002bfd50 map[0x2f67040:0xc002a85f20 0x2f67580:0xc00541a1e0 0x2f68000:0xc002a85e60 0x2f682a0:0xc002a85ec0 0x2f687e0:0xc00541a000 0x2f68a80:0xc00541a060 0x2f68d20:0xc00541a0c0 0x2f69500:0xc00541a240 0x2f697a0:0xc00541a2a0 0x2f93d40:0xc000f53380 0x2f940e0:0xc000f534a0 0x2f94bc0:0xc000c645a0 0x2f96520:0xc001cb6000 0x2f97000:0xc002504a80 0x2f97e80:0xc001507ec0] 0xc0000d37a0 0} I0327 10:43:20.833884 3868 gateway.go:519] Reconciling gateway with updates I0327 10:43:20.838991 3868 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 10:43:20.839071 3868 default_node_network_controller.go:943] Gateway and management port readiness took 68.391611ms I0327 10:43:20.839104 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 10:43:20.839123 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 10:43:20.839120 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 10:43:20.839139 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 10:43:20.839146 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 10:43:20.839159 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 10:43:20.839168 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 10:43:20.839180 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 10:43:20.839182 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 10:43:20.839189 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 10:43:20.839206 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 10:43:20.839206 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 10:43:20.839210 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 10:43:20.839217 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 10:43:20.839212 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 10:43:20.839218 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 10:43:20.839217 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 10:43:20.839218 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 10:43:20.839231 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 10:43:20.839235 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 10:43:20.839235 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 10:43:20.839244 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 10:43:20.839244 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 10:43:20.839258 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 10:43:20.839256 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 10:43:20.839205 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 10:43:20.839195 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 10:43:20.839286 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 10:43:20.839286 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 10:43:20.839288 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 10:43:20.839305 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 10:43:20.839307 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 10:43:20.839318 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 10:43:20.839321 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 10:43:20.839328 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 10:43:20.839195 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 10:43:20.839343 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 10:43:20.839187 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 10:43:20.839352 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 10:43:20.839361 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 10:43:20.839307 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 10:43:20.839368 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 10:43:20.839377 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 10:43:20.839292 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 10:43:20.839393 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 10:43:20.839399 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 10:43:20.839403 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 10:43:20.839321 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 10:43:20.839396 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 10:43:20.839429 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 10:43:20.839330 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 10:43:20.839434 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 10:43:20.839442 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 10:43:20.839443 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 10:43:20.839450 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 10:43:20.839449 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 10:43:20.839450 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 10:43:20.839460 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 10:43:20.839460 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 10:43:20.839466 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 10:43:20.839470 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 10:43:20.839475 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 10:43:20.839476 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 10:43:20.839452 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 10:43:20.839486 3868 port_claim.go:71] Svc openshift-ingress/router-default: port 32051 is already open, no action needed I0327 10:43:20.839491 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 10:43:20.839495 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 10:43:20.839496 3868 port_claim.go:71] Svc openshift-ingress/router-default: port 32225 is already open, no action needed I0327 10:43:20.839430 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 10:43:20.839503 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 10:43:20.839508 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 10:43:20.839276 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 10:43:20.839515 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 10:43:20.839517 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 10:43:20.839363 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 10:43:20.839522 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 10:43:20.839530 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 10:43:20.839354 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 10:43:20.839376 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 10:43:20.839195 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 10:43:20.839548 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 10:43:20.839374 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 10:43:20.839563 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 10:43:20.839447 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 10:43:20.839580 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 10:43:20.839208 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 10:43:20.839494 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 10:43:20.839522 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 10:43:20.839351 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 10:43:20.839185 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 10:43:20.839535 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 10:43:20.839699 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 10:43:20.839730 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 10:43:20.839230 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 10:43:20.839747 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 10:43:20.839752 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 10:43:20.839612 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 10:43:20.839800 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 10:43:20.839818 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 10:43:20.839208 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 10:43:20.839433 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 10:43:20.840686 3868 node_ip_handler_linux.go:160] Node IP manager is running W0327 10:43:20.874956 3868 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 10:43:20.875126 3868 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 10:43:20.875258 3868 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0327 10:43:20.883249 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 10:43:20.883270 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 10:43:20.883278 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 10:43:20.883284 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 10:43:20.883290 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 10:43:20.883297 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 10:43:20.883302 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 10:43:20.883310 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 10:43:20.883316 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 10:43:20.883321 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 10:43:20.883326 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 10:43:20.883332 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 10:43:20.883347 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 10:43:20.883363 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 10:43:20.883401 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 10:43:20.883415 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 10:43:20.883422 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 10:43:20.883428 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 10:43:20.883435 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 10:43:20.920353 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 10:43:20.920362 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 10:43:20.920377 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 10:43:20.920385 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 10:43:20.920393 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 10:43:20.920400 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 10:43:20.920408 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 10:43:20.920416 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 10:43:20.920422 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 10:43:20.920428 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 10:43:20.920434 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 10:43:20.920442 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 10:43:20.957650 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 10:43:20.957674 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 10:43:22.801315 3868 default_node_network_controller.go:1364] Starting node controller node sync W0327 10:43:22.801366 3868 egressip_healthcheck.go:76] Health checking using insecure connection I0327 10:43:22.801425 3868 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0327 10:43:22.812847 3868 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.510673ms I0327 10:43:22.812977 3868 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 10:43:22.813230 3868 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 10:43:22.813338 3868 egressservice_node.go:183] Starting Egress Services Controller I0327 10:43:22.813351 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 10:43:22.813361 3868 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 10:43:22.813367 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 10:43:22.813372 3868 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 10:43:22.813377 3868 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 10:43:22.813381 3868 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 10:43:22.813386 3868 egressservice_node.go:197] Repairing Egress Services I0327 10:43:22.823474 3868 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 10:43:22.823488 3868 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 10:43:22.823649 3868 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 10:43:22.823662 3868 link_network_manager.go:117] Link manager is running I0327 10:43:22.823674 3868 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 10:43:22.823739 3868 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 10:43:22.824057 3868 vrf_manager.go:128] VRF manager is running I0327 10:43:22.824147 3868 node_controller_manager.go:437] Removing flows to drop GARP I0327 10:43:22.824157 3868 gateway.go:519] Reconciling gateway with updates I0327 10:43:22.853566 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 10:43:22.853582 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 10:43:22.853588 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 10:43:22.853569 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 10:43:22.853596 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 10:43:22.853603 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 10:43:22.853605 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 10:43:22.853609 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 10:43:22.853611 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 10:43:22.853611 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 10:43:22.853612 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 10:43:22.853617 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 10:43:22.853624 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 10:43:22.853628 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 10:43:22.853582 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 10:43:22.853633 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 10:43:22.853638 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 10:43:22.853640 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 10:43:22.853649 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 10:43:22.853648 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 10:43:22.853654 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 10:43:22.853613 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 10:43:22.853658 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 10:43:22.853667 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 10:43:22.853669 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 10:43:22.853600 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 10:43:22.853671 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 10:43:22.853679 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 10:43:22.853684 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 10:43:22.853650 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 10:43:22.853692 3868 port_claim.go:71] Svc openshift-ingress/router-default: port 32051 is already open, no action needed I0327 10:43:22.853694 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 10:43:22.853592 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 10:43:22.853706 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 10:43:22.853727 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 10:43:22.853736 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 10:43:22.853735 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 10:43:22.853592 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 10:43:22.853740 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 10:43:22.853576 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 10:43:22.853753 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 10:43:22.853760 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 10:43:22.853761 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 10:43:22.853765 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 10:43:22.853768 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 10:43:22.853742 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 10:43:22.853728 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 10:43:22.853779 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853780 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 10:43:22.853780 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 10:43:22.853787 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 10:43:22.853789 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853795 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 10:43:22.853801 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 10:43:22.853802 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 10:43:22.853801 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 10:43:22.853794 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 10:43:22.853812 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 10:43:22.853807 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853815 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 10:43:22.853838 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 10:43:22.853839 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 10:43:22.853838 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 10:43:22.853846 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 10:43:22.853856 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 10:43:22.853854 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 10:43:22.853857 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 10:43:22.853866 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 10:43:22.853628 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 10:43:22.853875 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 10:43:22.853857 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 10:43:22.853899 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 10:43:22.853907 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853907 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 10:43:22.853911 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853845 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 10:43:22.853813 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 10:43:22.853944 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 10:43:22.853594 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 10:43:22.853982 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 10:43:22.853695 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 10:43:22.853993 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 10:43:22.854012 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 10:43:22.853638 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 10:43:22.854084 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 10:43:22.853782 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 10:43:22.854146 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853707 3868 port_claim.go:71] Svc openshift-ingress/router-default: port 32225 is already open, no action needed I0327 10:43:22.853627 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 10:43:22.854200 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 10:43:22.853632 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 10:43:22.854225 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 10:43:22.853643 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 10:43:22.853662 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 10:43:22.854250 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 10:43:22.853670 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 10:43:22.853681 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853696 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 10:43:22.853745 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 10:43:22.854285 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 10:43:22.853737 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 10:43:22.854307 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 10:43:22.853740 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 10:43:22.854334 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 10:43:22.853763 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 10:43:22.853776 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 10:43:22.853782 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 10:43:22.853616 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 10:43:22.854400 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 10:43:22.853601 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 10:43:22.854407 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 10:43:22.853781 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 10:43:22.853786 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 10:43:22.854416 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 10:43:22.853790 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 10:43:22.854423 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 10:43:22.853785 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 10:43:22.854445 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 10:43:22.853792 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 10:43:22.853796 3868 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 10:43:22.854484 3868 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 10:43:22.890808 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 10:43:22.890825 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 10:43:22.890832 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 10:43:22.890837 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 10:43:22.890844 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 10:43:22.909330 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 10:43:22.909348 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 10:43:22.909360 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 10:43:22.909368 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 10:43:22.909373 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 10:43:22.909379 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 10:43:22.909386 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 10:43:22.909391 3868 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 10:43:24.289044 3868 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-nxm5x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:43:24.289113 3868 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-nxm5x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:43:24.416296 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-4.ec2.internal I0327 10:43:24.416488 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-4.ec2.internal: 198.43µs I0327 10:43:24.454920 3868 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7d555f8896-kw7fv] creating logical port openshift-image-registry_image-registry-7d555f8896-kw7fv for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:24.455047 3868 kube.go:256] Updating pod openshift-image-registry/image-registry-7d555f8896-kw7fv I0327 10:43:24.481302 3868 pod.go:62] [openshift-image-registry/image-registry-7d555f8896-kw7fv] pod update took 26.284189ms I0327 10:43:24.481325 3868 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7d555f8896-kw7fv] addLogicalPort annotation time took 26.313153ms I0327 10:43:24.482685 3868 pods.go:271] [openshift-image-registry/image-registry-7d555f8896-kw7fv] addLogicalPort took 27.778476ms, libovsdb time 1.069401ms I0327 10:43:24.498735 3868 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-mt6bq] creating logical port openshift-dns_dns-default-mt6bq for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:24.498851 3868 kube.go:256] Updating pod openshift-dns/dns-default-mt6bq I0327 10:43:24.498915 3868 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-j6xz4 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:24.498999 3868 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4 I0327 10:43:24.516659 3868 pod.go:62] [openshift-dns/dns-default-mt6bq] pod update took 17.834046ms I0327 10:43:24.516684 3868 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-mt6bq] addLogicalPort annotation time took 17.860392ms I0327 10:43:24.516750 3868 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4] pod update took 17.76976ms I0327 10:43:24.516771 3868 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4] addLogicalPort annotation time took 17.794165ms I0327 10:43:24.518082 3868 pods.go:271] [openshift-dns/dns-default-mt6bq] addLogicalPort took 19.377269ms, libovsdb time 1.040576ms I0327 10:43:24.518481 3868 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4] addLogicalPort took 19.576708ms, libovsdb time 1.409724ms I0327 10:43:24.518563 3868 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-w8sv8] creating logical port openshift-ingress-canary_ingress-canary-w8sv8 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:24.518646 3868 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-w8sv8 I0327 10:43:24.531533 3868 pod.go:62] [openshift-ingress-canary/ingress-canary-w8sv8] pod update took 12.911872ms I0327 10:43:24.531556 3868 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-w8sv8] addLogicalPort annotation time took 12.939051ms I0327 10:43:24.532444 3868 pods.go:271] [openshift-ingress-canary/ingress-canary-w8sv8] addLogicalPort took 13.894862ms, libovsdb time 625.027µs I0327 10:43:24.890547 3868 cni.go:403] [openshift-ingress-canary/ingress-canary-w8sv8 7309bf95ed3b098bd8a9e80499efe435569c654d38b60d41ffdbfbd26ae738ab network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-w8sv8 7309bf95ed3b098bd8a9e80499efe435569c654d38b60d41ffdbfbd26ae738ab network default NAD default] I0327 10:43:24.901903 3868 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4 624f993ea167c4ec7bf30517720c538de94651323995fd1ad765d7c552e9061e network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4 624f993ea167c4ec7bf30517720c538de94651323995fd1ad765d7c552e9061e network default NAD default] I0327 10:43:24.916369 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-w8sv8, hostIfaceName: 7309bf95ed3b098, network: default, NAD default, SandboxID: "7309bf95ed3b098bd8a9e80499efe435569c654d38b60d41ffdbfbd26ae738ab", PCI device ID: , UID: "6a8b55df-9402-4dd2-b1b7-4d998c5fc385", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0327 10:43:24.917630 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-j6xz4, hostIfaceName: 624f993ea167c4e, network: default, NAD default, SandboxID: "624f993ea167c4ec7bf30517720c538de94651323995fd1ad765d7c552e9061e", PCI device ID: , UID: "8cfa876e-23e1-475b-af56-43057e6c9efb", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0327 10:43:24.967451 3868 cni.go:424] [openshift-ingress-canary/ingress-canary-w8sv8 7309bf95ed3b098bd8a9e80499efe435569c654d38b60d41ffdbfbd26ae738ab network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-w8sv8 7309bf95ed3b098bd8a9e80499efe435569c654d38b60d41ffdbfbd26ae738ab network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:6c:be:4a:ef:a6\",\"name\":\"7309bf95ed3b098\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0788c51e-f874-4287-9d57-260547471078\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:24.968179 3868 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4 624f993ea167c4ec7bf30517720c538de94651323995fd1ad765d7c552e9061e network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-j6xz4 624f993ea167c4ec7bf30517720c538de94651323995fd1ad765d7c552e9061e network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:eb:e6:0b:d8:2e\",\"name\":\"624f993ea167c4e\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/605c2ef2-1713-4ff4-bdd4-62f2613c49cc\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:25.283322 3868 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-nxm5x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:43:25.283364 3868 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-nxm5x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:43:26.088021 3868 cni.go:403] [openshift-network-diagnostics/network-check-target-bh89c 7cb6c54b31fe537b03656b091a2f564e4130e1b6c301fe6de0296bcb73f4ce23 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-bh89c 7cb6c54b31fe537b03656b091a2f564e4130e1b6c301fe6de0296bcb73f4ce23 network default NAD default] I0327 10:43:26.097581 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-bh89c, hostIfaceName: 7cb6c54b31fe537, network: default, NAD default, SandboxID: "7cb6c54b31fe537b03656b091a2f564e4130e1b6c301fe6de0296bcb73f4ce23", PCI device ID: , UID: "23f4259f-81e2-47b9-a300-3447d709aa39", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0327 10:43:26.137336 3868 cni.go:424] [openshift-network-diagnostics/network-check-target-bh89c 7cb6c54b31fe537b03656b091a2f564e4130e1b6c301fe6de0296bcb73f4ce23 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-bh89c 7cb6c54b31fe537b03656b091a2f564e4130e1b6c301fe6de0296bcb73f4ce23 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:af:67:49:db:07\",\"name\":\"7cb6c54b31fe537\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5960132e-fd78-4911-a33e-ce50d24674e1\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:27.691999 3868 egressfirewall.go:628] Syncing node "ip-10-0-1-188.ec2.internal" for egress firewall I0327 10:43:27.692004 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-188.ec2.internal I0327 10:43:27.692029 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-188.ec2.internal: 31.314µs I0327 10:43:27.692058 3868 default_network_controller.go:1008] Node "ip-10-0-1-188.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.446214 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-12.ec2.internal in network "default" I0327 10:43:32.446240 3868 node_tracker.go:220] Node ip-10-0-0-12.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0327 10:43:32.446250 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:43:32.446261 3868 egressfirewall.go:628] Syncing node "ip-10-0-0-12.ec2.internal" for egress firewall I0327 10:43:32.446269 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 24.357µs E0327 10:43:32.446352 3868 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-12.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0327 10:43:32.446372 3868 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-12.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-12.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-12.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-12.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-12.ec2.internal, error parsing annotation for node ip-10-0-0-12.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0327 10:43:32.460415 3868 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-12.ec2.internal annotation: unexpected end of JSON input E0327 10:43:32.460446 3868 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-12.ec2.internal annotation: unexpected end of JSON input I0327 10:43:32.460464 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0327 10:43:32.460513 3868 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-12.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-12.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-12.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-12.ec2.internal" I0327 10:43:32.474440 3868 egressfirewall.go:628] Syncing node "ip-10-0-1-188.ec2.internal" for egress firewall I0327 10:43:32.474461 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-188.ec2.internal I0327 10:43:32.474471 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-188.ec2.internal: 12.69µs I0327 10:43:32.474495 3868 default_network_controller.go:1008] Node "ip-10-0-1-188.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.475103 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0327 10:43:32.475162 3868 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-12.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-12.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-12.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-12.ec2.internal" I0327 10:43:32.492764 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.492783 3868 egressfirewall.go:628] Syncing node "ip-10-0-0-12.ec2.internal" for egress firewall I0327 10:43:32.492788 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:43:32.492802 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 18.581µs E0327 10:43:32.492852 3868 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-12.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-12.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-12.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-12.ec2.internal" I0327 10:43:32.505972 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0327 10:43:32.676917 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:43:32.676935 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 23.546µs I0327 10:43:32.676934 3868 egressfirewall.go:628] Syncing node "ip-10-0-0-12.ec2.internal" for egress firewall I0327 10:43:32.677036 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.696319 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.709214 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-188.ec2.internal in network "default" I0327 10:43:32.709241 3868 default_network_controller.go:1008] Node "ip-10-0-1-188.ec2.internal" in remote zone "ip-10-0-1-188.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.709262 3868 node_tracker.go:235] Node ip-10-0-1-188.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-188.ec2.internal" I0327 10:43:32.709278 3868 node_tracker.go:169] Node ip-10-0-1-188.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:32.709297 3868 services_controller.go:574] Full service sync requested for network=default I0327 10:43:32.760165 3868 default_network_controller.go:1008] Node "ip-10-0-1-188.ec2.internal" in remote zone "ip-10-0-1-188.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.837599 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-188.ec2.internal in network "default" I0327 10:43:32.837843 3868 node_tracker.go:169] Node ip-10-0-1-188.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:32.837857 3868 services_controller.go:574] Full service sync requested for network=default I0327 10:43:32.837622 3868 default_network_controller.go:1008] Node "ip-10-0-1-188.ec2.internal" in remote zone "ip-10-0-1-188.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:32.838511 3868 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-188.ec2.internal for the network default I0327 10:43:32.839511 3868 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-188.ec2.internal" on network "default" took: 1.015603ms I0327 10:43:32.890441 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0327 10:43:33.345947 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 10:43:33.379113 3868 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 10:43:35.126235 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:35.393781 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-188.ec2.internal I0327 10:43:35.393805 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-188.ec2.internal: 31.685µs I0327 10:43:35.393783 3868 egressfirewall.go:628] Syncing node "ip-10-0-1-188.ec2.internal" for egress firewall I0327 10:43:35.393817 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-188.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 10:43:35.393832 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-188.ec2.internal", IP: 10.0.1.188 to PMTUD blocking rules I0327 10:43:35.393846 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-188.ec2.internal in network "default" I0327 10:43:35.393896 3868 node_tracker.go:169] Node ip-10-0-1-188.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:35.393912 3868 services_controller.go:574] Full service sync requested for network=default W0327 10:43:35.394247 3868 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0327 10:43:35.394249 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 10:43:35.400357 3868 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-188.ec2.internal. Updating no re-route policies I0327 10:43:35.411492 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-188.ec2.internal", IP: 10.0.1.188 to PMTUD blocking rules I0327 10:43:35.411506 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-188.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 10:43:39.521136 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-188.ec2.internal I0327 10:43:39.521168 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-188.ec2.internal: 39.98µs I0327 10:43:40.202793 3868 cni.go:403] [kube-system/global-pull-secret-syncer-4vkbk 26eb769d8b0c8993f1a258cb0f30210cfc9d6f305e79e2a646f5db14619f8931 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-4vkbk 26eb769d8b0c8993f1a258cb0f30210cfc9d6f305e79e2a646f5db14619f8931 network default NAD default] I0327 10:43:40.211840 3868 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-4vkbk, hostIfaceName: 26eb769d8b0c899, network: default, NAD default, SandboxID: "26eb769d8b0c8993f1a258cb0f30210cfc9d6f305e79e2a646f5db14619f8931", PCI device ID: , UID: "8dc8a78d-9e29-45b7-85c8-f103b5fd3660", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0327 10:43:40.252959 3868 cni.go:424] [kube-system/global-pull-secret-syncer-4vkbk 26eb769d8b0c8993f1a258cb0f30210cfc9d6f305e79e2a646f5db14619f8931 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-4vkbk 26eb769d8b0c8993f1a258cb0f30210cfc9d6f305e79e2a646f5db14619f8931 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:96:fe:61:49:6f\",\"name\":\"26eb769d8b0c899\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10f4f2ff-1da2-471e-906c-c583b264177a\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:42.548121 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0327 10:43:45.877930 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 10:43:46.333346 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 10:43:49.078261 3868 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-0-12.ec2.internal I0327 10:43:49.078282 3868 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-0-12.ec2.internal I0327 10:43:49.078364 3868 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-0-12.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-12.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-12.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-12.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-12.ec2.internal I0327 10:43:49.138227 3868 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-896fh for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:49.138340 3868 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh I0327 10:43:49.166297 3868 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh] pod update took 27.975274ms I0327 10:43:49.166314 3868 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh] addLogicalPort annotation time took 27.997679ms I0327 10:43:49.167264 3868 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh] addLogicalPort took 29.048311ms, libovsdb time 697.757µs I0327 10:43:49.239290 3868 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-4whrj for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:49.239365 3868 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj I0327 10:43:49.241321 3868 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-2qc77 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:49.241427 3868 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77 I0327 10:43:49.258923 3868 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj] pod update took 19.575176ms I0327 10:43:49.258947 3868 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj] addLogicalPort annotation time took 19.601351ms I0327 10:43:49.259939 3868 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj] addLogicalPort took 20.659578ms, libovsdb time 745.034µs I0327 10:43:49.266417 3868 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77] pod update took 25.006456ms I0327 10:43:49.266435 3868 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77] addLogicalPort annotation time took 25.02842ms I0327 10:43:49.267336 3868 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77] addLogicalPort took 26.039339ms, libovsdb time 668.957µs I0327 10:43:49.267376 3868 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-6jqfw] creating logical port openshift-console-operator_console-operator-8777bd9bb-6jqfw for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:49.267423 3868 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-6jqfw I0327 10:43:49.288242 3868 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-6jqfw] pod update took 20.831686ms I0327 10:43:49.288264 3868 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-6jqfw] addLogicalPort annotation time took 20.856026ms I0327 10:43:49.289108 3868 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-6jqfw] addLogicalPort took 21.736411ms, libovsdb time 557.239µs I0327 10:43:49.292809 3868 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-74dcd44967-wtg8q] creating logical port openshift-ingress_router-default-74dcd44967-wtg8q for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:49.292879 3868 kube.go:256] Updating pod openshift-ingress/router-default-74dcd44967-wtg8q I0327 10:43:49.321202 3868 pod.go:62] [openshift-ingress/router-default-74dcd44967-wtg8q] pod update took 28.348569ms I0327 10:43:49.321223 3868 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-74dcd44967-wtg8q] addLogicalPort annotation time took 28.371244ms I0327 10:43:49.322340 3868 pods.go:271] [openshift-ingress/router-default-74dcd44967-wtg8q] addLogicalPort took 29.538364ms, libovsdb time 799.155µs I0327 10:43:49.335615 3868 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-fx4qd for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:49.335743 3868 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd I0327 10:43:49.353802 3868 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd] pod update took 18.106585ms I0327 10:43:49.353824 3868 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd] addLogicalPort annotation time took 18.133007ms I0327 10:43:49.354841 3868 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd] addLogicalPort took 19.276444ms, libovsdb time 712.602µs I0327 10:43:49.630180 3868 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj 4fda005026926dd7d09c100e71625b502daabde51dcbe547a9a21021cbbe432e network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj 4fda005026926dd7d09c100e71625b502daabde51dcbe547a9a21021cbbe432e network default NAD default] I0327 10:43:49.639627 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-4whrj, hostIfaceName: 4fda005026926dd, network: default, NAD default, SandboxID: "4fda005026926dd7d09c100e71625b502daabde51dcbe547a9a21021cbbe432e", PCI device ID: , UID: "ef79dd9f-e17d-4db8-afa1-8fcfa8f31d3b", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0327 10:43:49.659337 3868 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-6jqfw fc975af88dfd670cb582f5b1c17b50eb7e12e666cc00cf3c802fe32a34ecc356 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-6jqfw fc975af88dfd670cb582f5b1c17b50eb7e12e666cc00cf3c802fe32a34ecc356 network default NAD default] I0327 10:43:49.659599 3868 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77 da563d46e81b249b285c41532d9fa25a6e84894fb9c1acbcc411f4e718a3864f network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77 da563d46e81b249b285c41532d9fa25a6e84894fb9c1acbcc411f4e718a3864f network default NAD default] I0327 10:43:49.672950 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-6jqfw, hostIfaceName: fc975af88dfd670, network: default, NAD default, SandboxID: "fc975af88dfd670cb582f5b1c17b50eb7e12e666cc00cf3c802fe32a34ecc356", PCI device ID: , UID: "61fbd321-a462-4ca3-b87b-2557697062c5", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0327 10:43:49.673464 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-2qc77, hostIfaceName: da563d46e81b249, network: default, NAD default, SandboxID: "da563d46e81b249b285c41532d9fa25a6e84894fb9c1acbcc411f4e718a3864f", PCI device ID: , UID: "601f5364-88a2-4edf-9138-4529b0205c75", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0327 10:43:49.688372 3868 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj 4fda005026926dd7d09c100e71625b502daabde51dcbe547a9a21021cbbe432e network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-4whrj 4fda005026926dd7d09c100e71625b502daabde51dcbe547a9a21021cbbe432e network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:83:e8:d6:bb:77\",\"name\":\"4fda005026926dd\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/15b2a15e-f4fd-47e9-b97f-ea9189a9cdbc\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:49.714485 3868 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-6jqfw fc975af88dfd670cb582f5b1c17b50eb7e12e666cc00cf3c802fe32a34ecc356 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-6jqfw fc975af88dfd670cb582f5b1c17b50eb7e12e666cc00cf3c802fe32a34ecc356 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:ba:99:eb:ac:dd\",\"name\":\"fc975af88dfd670\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e05c0b8b-5d69-4141-8ddc-a97cdfb59261\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:49.729277 3868 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77 da563d46e81b249b285c41532d9fa25a6e84894fb9c1acbcc411f4e718a3864f network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-2qc77 da563d46e81b249b285c41532d9fa25a6e84894fb9c1acbcc411f4e718a3864f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:46:0a:1f:f1:32\",\"name\":\"da563d46e81b249\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0b9754e-c393-4df4-a393-a51d2d40c37b\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:50.619594 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-q7vjz I0327 10:43:50.619630 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-q7vjz I0327 10:43:50.619677 3868 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-q7vjz, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-q7vjz: suppressed error logged: pod openshift-multus/network-metrics-daemon-q7vjz: no pod IPs found I0327 10:43:50.619602 3868 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-5whrf I0327 10:43:50.619688 3868 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-5whrf I0327 10:43:50.619706 3868 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-5whrf, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5whrf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5whrf: no pod IPs found I0327 10:43:53.372177 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:43:53.372180 3868 egressfirewall.go:628] Syncing node "ip-10-0-0-12.ec2.internal" for egress firewall I0327 10:43:53.372195 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 23.646µs W0327 10:43:53.395085 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 10:43:53.404660 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:53.863467 3868 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 10:43:53.864389 3868 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 903.758µs I0327 10:43:53.898998 3868 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 10:43:54.061951 3868 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-dsd66 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:54.062103 3868 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66 I0327 10:43:54.082181 3868 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66] pod update took 20.112162ms I0327 10:43:54.082206 3868 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66] addLogicalPort annotation time took 20.138206ms I0327 10:43:54.083266 3868 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66] addLogicalPort took 21.334772ms, libovsdb time 776.099µs I0327 10:43:54.412755 3868 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66 93b42b557fbc800c9e0582d21709bf8f203fce8c854583ff1947c52f4e47c06e network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66 93b42b557fbc800c9e0582d21709bf8f203fce8c854583ff1947c52f4e47c06e network default NAD default] I0327 10:43:54.421250 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-dsd66, hostIfaceName: 93b42b557fbc800, network: default, NAD default, SandboxID: "93b42b557fbc800c9e0582d21709bf8f203fce8c854583ff1947c52f4e47c06e", PCI device ID: , UID: "ed6f8f10-193c-4dae-828e-7b24876eb9d7", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0327 10:43:54.462292 3868 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66 93b42b557fbc800c9e0582d21709bf8f203fce8c854583ff1947c52f4e47c06e network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dsd66 93b42b557fbc800c9e0582d21709bf8f203fce8c854583ff1947c52f4e47c06e network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:af:6f:34:82:c6\",\"name\":\"93b42b557fbc800\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/083a9add-c096-4f3c-a630-0d1f0ac626f3\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:55.681803 3868 namespace.go:98] [openshift-service-ca] adding namespace I0327 10:43:55.682673 3868 namespace.go:102] [openshift-service-ca] adding namespace took 841.22µs I0327 10:43:55.708481 3868 namespace.go:142] [openshift-service-ca] updating namespace I0327 10:43:57.375328 3868 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-prpzn] creating logical port openshift-service-ca_service-ca-f59fc589b-prpzn for pod on switch ip-10-0-2-4.ec2.internal I0327 10:43:57.375467 3868 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-prpzn I0327 10:43:57.390206 3868 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-prpzn] pod update took 14.763412ms I0327 10:43:57.390225 3868 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-prpzn] addLogicalPort annotation time took 14.787542ms I0327 10:43:57.391362 3868 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-prpzn] addLogicalPort took 16.049614ms, libovsdb time 853.821µs I0327 10:43:57.729950 3868 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-prpzn 14aaca7e330d813d6d11b279b262afd459c04c486a8564ade0405674c65bd6f0 network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-prpzn 14aaca7e330d813d6d11b279b262afd459c04c486a8564ade0405674c65bd6f0 network default NAD default] I0327 10:43:57.739163 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-prpzn, hostIfaceName: 14aaca7e330d813, network: default, NAD default, SandboxID: "14aaca7e330d813d6d11b279b262afd459c04c486a8564ade0405674c65bd6f0", PCI device ID: , UID: "aba1cf67-a57c-49b3-8ecc-3d280bc515d3", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0327 10:43:57.780239 3868 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-prpzn 14aaca7e330d813d6d11b279b262afd459c04c486a8564ade0405674c65bd6f0 network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-prpzn 14aaca7e330d813d6d11b279b262afd459c04c486a8564ade0405674c65bd6f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:f7:4c:95:03:42\",\"name\":\"14aaca7e330d813\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dbeeb22d-90ac-4595-880c-8e11bdf61a49\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:43:58.743355 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-12.ec2.internal in network "default" I0327 10:43:58.743407 3868 node_tracker.go:235] Node ip-10-0-0-12.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-12.ec2.internal" I0327 10:43:58.743408 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "ip-10-0-0-12.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:58.743428 3868 node_tracker.go:169] Node ip-10-0-0-12.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:58.743440 3868 services_controller.go:574] Full service sync requested for network=default W0327 10:43:58.743701 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0327 10:43:58.743851 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 10:43:58.829303 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "ip-10-0-0-12.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:58.859426 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-12.ec2.internal in network "default" I0327 10:43:58.859467 3868 default_network_controller.go:1008] Node "ip-10-0-0-12.ec2.internal" in remote zone "ip-10-0-0-12.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 10:43:58.859483 3868 node_tracker.go:169] Node ip-10-0-0-12.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:58.859494 3868 services_controller.go:574] Full service sync requested for network=default W0327 10:43:58.859568 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0327 10:43:58.859955 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 10:43:58.860183 3868 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-12.ec2.internal for the network default I0327 10:43:58.861352 3868 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-12.ec2.internal" on network "default" took: 1.182355ms I0327 10:43:59.909634 3868 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-12.ec2.internal in network "default" I0327 10:43:59.909663 3868 egressfirewall.go:628] Syncing node "ip-10-0-0-12.ec2.internal" for egress firewall I0327 10:43:59.909706 3868 node_tracker.go:169] Node ip-10-0-0-12.ec2.internal switch + router changed, syncing services in network "default" I0327 10:43:59.909640 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:43:59.909723 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-12.ec2.internal", IP: 10.0.0.12 to PMTUD blocking rules I0327 10:43:59.909735 3868 services_controller.go:574] Full service sync requested for network=default I0327 10:43:59.909743 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-12.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 10:43:59.909743 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 106.848µs W0327 10:43:59.909957 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0327 10:43:59.910128 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 10:43:59.915840 3868 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-12.ec2.internal. Updating no re-route policies I0327 10:43:59.925470 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-12.ec2.internal", IP: 10.0.0.12 to PMTUD blocking rules I0327 10:43:59.925485 3868 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-12.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 10:44:05.122921 3868 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh b423e191a5b34b71a7e4786adeea750ee8cd6e76aa30febc6a05ab85c01a5c96 network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh b423e191a5b34b71a7e4786adeea750ee8cd6e76aa30febc6a05ab85c01a5c96 network default NAD default] I0327 10:44:05.131180 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-896fh, hostIfaceName: b423e191a5b34b7, network: default, NAD default, SandboxID: "b423e191a5b34b71a7e4786adeea750ee8cd6e76aa30febc6a05ab85c01a5c96", PCI device ID: , UID: "bd4f90e8-1e4e-4066-a2fc-990cda07769c", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0327 10:44:05.173390 3868 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh b423e191a5b34b71a7e4786adeea750ee8cd6e76aa30febc6a05ab85c01a5c96 network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-896fh b423e191a5b34b71a7e4786adeea750ee8cd6e76aa30febc6a05ab85c01a5c96 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:2b:dc:f3:d4:73\",\"name\":\"b423e191a5b34b7\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/df371afb-52ef-4c09-84b2-8e4654ac8c45\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:06.843560 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0327 10:44:17.375119 3868 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:17.378135 3868 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:17.378147 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:17.378151 3868 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:17.384095 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 10:44:17.434267 3868 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-747b8fcfd7-nsslh IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-747b8fcfd7-nsslh: no pod IPs found I0327 10:44:17.489823 3868 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-4h2ls] creating logical port openshift-insights_insights-runtime-extractor-4h2ls for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:17.489916 3868 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-4h2ls I0327 10:44:17.508931 3868 pod.go:62] [openshift-insights/insights-runtime-extractor-4h2ls] pod update took 19.042278ms I0327 10:44:17.508951 3868 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-4h2ls] addLogicalPort annotation time took 19.065169ms I0327 10:44:17.510190 3868 pods.go:271] [openshift-insights/insights-runtime-extractor-4h2ls] addLogicalPort took 20.385207ms, libovsdb time 952.777µs I0327 10:44:17.785759 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:44:17.785785 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 31.115µs I0327 10:44:17.868872 3868 cni.go:403] [openshift-insights/insights-runtime-extractor-4h2ls 3aa00aa43d6c4c96e4c5517f2df338c33e861f8705e04f3da2d49d0b000138e8 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-4h2ls 3aa00aa43d6c4c96e4c5517f2df338c33e861f8705e04f3da2d49d0b000138e8 network default NAD default] I0327 10:44:17.877363 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-4h2ls, hostIfaceName: 3aa00aa43d6c4c9, network: default, NAD default, SandboxID: "3aa00aa43d6c4c96e4c5517f2df338c33e861f8705e04f3da2d49d0b000138e8", PCI device ID: , UID: "0692672f-91dc-4f04-bcd6-cdf2bdf9da7d", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0327 10:44:17.919743 3868 cni.go:424] [openshift-insights/insights-runtime-extractor-4h2ls 3aa00aa43d6c4c96e4c5517f2df338c33e861f8705e04f3da2d49d0b000138e8 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-4h2ls 3aa00aa43d6c4c96e4c5517f2df338c33e861f8705e04f3da2d49d0b000138e8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:f5:42:b7:67:ec\",\"name\":\"3aa00aa43d6c4c9\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e6ad3326-df1b-4ddb-bf80-f48464ad6c88\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:18.557889 3868 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 10:44:19.475321 3868 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d555f8896-kw7fv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:44:19.475367 3868 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7d555f8896-kw7fv I0327 10:44:19.476763 3868 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7d555f8896-kw7fv, ips: 10.128.0.6 I0327 10:44:19.476798 3868 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d555f8896-kw7fv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:44:19.484897 3868 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6c76bdd6f4-cfhnr] creating logical port openshift-image-registry_image-registry-6c76bdd6f4-cfhnr for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:19.484978 3868 kube.go:256] Updating pod openshift-image-registry/image-registry-6c76bdd6f4-cfhnr I0327 10:44:19.501212 3868 pod.go:62] [openshift-image-registry/image-registry-6c76bdd6f4-cfhnr] pod update took 16.25797ms I0327 10:44:19.501238 3868 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-6c76bdd6f4-cfhnr] addLogicalPort annotation time took 16.288116ms I0327 10:44:19.502432 3868 pods.go:271] [openshift-image-registry/image-registry-6c76bdd6f4-cfhnr] addLogicalPort took 17.566944ms, libovsdb time 885.397µs I0327 10:44:19.502475 3868 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d555f8896-kw7fv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:44:19.502487 3868 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d555f8896-kw7fv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:44:19.502867 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7d555f8896-kw7fv I0327 10:44:19.502896 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7d555f8896-kw7fv I0327 10:44:19.843143 3868 cni.go:403] [openshift-image-registry/image-registry-6c76bdd6f4-cfhnr d3ad04209653c6fad3fd9d89a4cbe8c3ab75aaed8069bd0da63aa43701cb1b49 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-6c76bdd6f4-cfhnr d3ad04209653c6fad3fd9d89a4cbe8c3ab75aaed8069bd0da63aa43701cb1b49 network default NAD default] I0327 10:44:19.851667 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6c76bdd6f4-cfhnr, hostIfaceName: d3ad04209653c6f, network: default, NAD default, SandboxID: "d3ad04209653c6fad3fd9d89a4cbe8c3ab75aaed8069bd0da63aa43701cb1b49", PCI device ID: , UID: "a1f9acea-06d3-4f73-bd0d-4f4820798fd0", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0327 10:44:19.902184 3868 cni.go:424] [openshift-image-registry/image-registry-6c76bdd6f4-cfhnr d3ad04209653c6fad3fd9d89a4cbe8c3ab75aaed8069bd0da63aa43701cb1b49 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-6c76bdd6f4-cfhnr d3ad04209653c6fad3fd9d89a4cbe8c3ab75aaed8069bd0da63aa43701cb1b49 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:06:13:92:13:cd\",\"name\":\"d3ad04209653c6f\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6caf6e61-ff56-48ad-94ab-366af7dcbe78\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:20.480004 3868 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 10:44:21.188644 3868 cni.go:403] [openshift-ingress/router-default-74dcd44967-wtg8q e9104d25a2923f2857e9c81181c0b842013998f282fcd24f509ef842f3a2f979 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-74dcd44967-wtg8q e9104d25a2923f2857e9c81181c0b842013998f282fcd24f509ef842f3a2f979 network default NAD default] I0327 10:44:21.197731 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-74dcd44967-wtg8q, hostIfaceName: e9104d25a2923f2, network: default, NAD default, SandboxID: "e9104d25a2923f2857e9c81181c0b842013998f282fcd24f509ef842f3a2f979", PCI device ID: , UID: "c4f10be5-f00d-4bc5-9602-989ca39d35ab", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0327 10:44:21.239975 3868 cni.go:424] [openshift-ingress/router-default-74dcd44967-wtg8q e9104d25a2923f2857e9c81181c0b842013998f282fcd24f509ef842f3a2f979 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-74dcd44967-wtg8q e9104d25a2923f2857e9c81181c0b842013998f282fcd24f509ef842f3a2f979 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:a1:4c:84:95:af\",\"name\":\"e9104d25a2923f2\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6aa0d0bf-9d0a-4687-8716-d44c358a39ca\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:21.521124 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:21.521150 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:21.521180 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:21.521190 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:21.521197 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default I0327 10:44:21.523947 3868 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd fb5541520ff7a5dad92f14ef6dcc76c63de616246f3ed26619ed08833be0ecd5 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd fb5541520ff7a5dad92f14ef6dcc76c63de616246f3ed26619ed08833be0ecd5 network default NAD default] I0327 10:44:21.534468 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-fx4qd, hostIfaceName: fb5541520ff7a5d, network: default, NAD default, SandboxID: "fb5541520ff7a5dad92f14ef6dcc76c63de616246f3ed26619ed08833be0ecd5", PCI device ID: , UID: "3c03b827-9bee-403b-9369-c9109d8e1a92", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0327 10:44:21.577143 3868 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd fb5541520ff7a5dad92f14ef6dcc76c63de616246f3ed26619ed08833be0ecd5 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-fx4qd fb5541520ff7a5dad92f14ef6dcc76c63de616246f3ed26619ed08833be0ecd5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:98:38:96:31:c4\",\"name\":\"fb5541520ff7a5d\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb53656d-f4c8-40c0-b1eb-f9190a4b9923\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:21.585398 3868 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:21.585414 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:21.585418 3868 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:22.000369 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.000393 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.000400 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.000478 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:22.000489 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:22.026958 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:22.026981 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:22.027084 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.027100 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.027105 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.501218 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.501237 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.501243 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 10:44:22.503951 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0327 10:44:22.503967 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default I0327 10:44:22.618583 3868 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0327 10:44:27.028015 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0327 10:44:28.777734 3868 cni.go:403] [openshift-dns/dns-default-mt6bq da17c730b61aebb9836645056e3874cf0afceb985ff26d816faefc443b9a170d network default NAD default] ADD starting CNI request [openshift-dns/dns-default-mt6bq da17c730b61aebb9836645056e3874cf0afceb985ff26d816faefc443b9a170d network default NAD default] I0327 10:44:28.786158 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-mt6bq, hostIfaceName: da17c730b61aebb, network: default, NAD default, SandboxID: "da17c730b61aebb9836645056e3874cf0afceb985ff26d816faefc443b9a170d", PCI device ID: , UID: "d0c47b3d-b557-4d94-93f9-c6aae37c4582", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0327 10:44:28.829902 3868 cni.go:424] [openshift-dns/dns-default-mt6bq da17c730b61aebb9836645056e3874cf0afceb985ff26d816faefc443b9a170d network default NAD default] ADD finished CNI request [openshift-dns/dns-default-mt6bq da17c730b61aebb9836645056e3874cf0afceb985ff26d816faefc443b9a170d network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:c1:a2:37:18:9d\",\"name\":\"da17c730b61aebb\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d332c868-bd82-46c4-bec1-d08d5e2bce5e\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:31.569924 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:31.569951 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:31.569958 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:31.569972 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:31.569977 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:31.569981 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:33.993516 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:33.993547 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 10:44:35.209056 3868 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:35.209156 3868 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0327 10:44:35.249428 3868 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 40.305969ms I0327 10:44:35.249457 3868 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 40.338173ms I0327 10:44:35.250959 3868 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 41.918433ms, libovsdb time 1.02263ms I0327 10:44:36.164320 3868 cni.go:403] [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default] I0327 10:44:36.172912 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 2a127da69fbe6de, network: default, NAD default, SandboxID: "2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb", PCI device ID: , UID: "5a496997-9374-46c4-91b8-347e52bf9769", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0327 10:44:36.216481 3868 cni.go:424] [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:56:31:ab:e4:13\",\"name\":\"2a127da69fbe6de\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/329fa222-c1b8-48de-bd28-9076339b9ff3\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:44:37.176465 3868 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-66d9794bdc-hszlc] creating logical port openshift-monitoring_thanos-querier-66d9794bdc-hszlc for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:37.176584 3868 kube.go:256] Updating pod openshift-monitoring/thanos-querier-66d9794bdc-hszlc I0327 10:44:37.195747 3868 pod.go:62] [openshift-monitoring/thanos-querier-66d9794bdc-hszlc] pod update took 19.190774ms I0327 10:44:37.195775 3868 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-66d9794bdc-hszlc] addLogicalPort annotation time took 19.223092ms I0327 10:44:37.197163 3868 pods.go:271] [openshift-monitoring/thanos-querier-66d9794bdc-hszlc] addLogicalPort took 20.711673ms, libovsdb time 1.007433ms I0327 10:44:37.563363 3868 cni.go:403] [openshift-monitoring/thanos-querier-66d9794bdc-hszlc 0cc91fe5da938ad16690d15504fd14f107e54ceb492143b02ac5c03b173cb805 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-66d9794bdc-hszlc 0cc91fe5da938ad16690d15504fd14f107e54ceb492143b02ac5c03b173cb805 network default NAD default] I0327 10:44:37.571894 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-66d9794bdc-hszlc, hostIfaceName: 0cc91fe5da938ad, network: default, NAD default, SandboxID: "0cc91fe5da938ad16690d15504fd14f107e54ceb492143b02ac5c03b173cb805", PCI device ID: , UID: "7b2accd6-01ee-4f17-a4cb-7ec7990c73aa", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0327 10:44:37.615504 3868 cni.go:424] [openshift-monitoring/thanos-querier-66d9794bdc-hszlc 0cc91fe5da938ad16690d15504fd14f107e54ceb492143b02ac5c03b173cb805 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-66d9794bdc-hszlc 0cc91fe5da938ad16690d15504fd14f107e54ceb492143b02ac5c03b173cb805 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:94:1e:10:93:b3\",\"name\":\"0cc91fe5da938ad\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7fe1248-f7fa-49b0-bb47-541c797fcf0d\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:38.585256 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:44:38.585279 3868 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:44:38.585284 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:44:38.663705 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:44:38.663776 3868 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:44:38.663785 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 10:44:38.786095 3868 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-6bdffd7594-8z4vw] creating logical port openshift-monitoring_metrics-server-6bdffd7594-8z4vw for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:38.786203 3868 kube.go:256] Updating pod openshift-monitoring/metrics-server-6bdffd7594-8z4vw I0327 10:44:38.803777 3868 pod.go:62] [openshift-monitoring/metrics-server-6bdffd7594-8z4vw] pod update took 17.592033ms I0327 10:44:38.803802 3868 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-6bdffd7594-8z4vw] addLogicalPort annotation time took 17.621578ms I0327 10:44:38.805126 3868 pods.go:271] [openshift-monitoring/metrics-server-6bdffd7594-8z4vw] addLogicalPort took 19.045233ms, libovsdb time 905.469µs I0327 10:44:39.148398 3868 cni.go:403] [openshift-monitoring/metrics-server-6bdffd7594-8z4vw b1db0fa80732e864fbff8683eabcc654e95384d66e8bdcba0789bcdd5d61dfed network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-6bdffd7594-8z4vw b1db0fa80732e864fbff8683eabcc654e95384d66e8bdcba0789bcdd5d61dfed network default NAD default] I0327 10:44:39.157062 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-6bdffd7594-8z4vw, hostIfaceName: b1db0fa80732e86, network: default, NAD default, SandboxID: "b1db0fa80732e864fbff8683eabcc654e95384d66e8bdcba0789bcdd5d61dfed", PCI device ID: , UID: "d7896ef8-2873-46c8-9c3d-2764bbab7c27", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0327 10:44:39.204284 3868 cni.go:424] [openshift-monitoring/metrics-server-6bdffd7594-8z4vw b1db0fa80732e864fbff8683eabcc654e95384d66e8bdcba0789bcdd5d61dfed network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-6bdffd7594-8z4vw b1db0fa80732e864fbff8683eabcc654e95384d66e8bdcba0789bcdd5d61dfed network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:61:a1:2a:b8:a3\",\"name\":\"b1db0fa80732e86\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9c526f17-2974-4924-b751-34049e5d5bb8\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:39.563581 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:39.563603 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:39.616959 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:39.616984 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:40.550175 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:40.550196 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:40.550201 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 10:44:41.082884 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0327 10:44:41.508773 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 10:44:42.112034 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 10:44:42.112053 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 10:44:42.605297 3868 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:42.605315 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:42.605320 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:42.605324 3868 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:42.640445 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 10:44:42.683759 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 10:44:42.749412 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 10:44:42.749435 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 10:44:43.702123 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:43.702141 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:43.702146 3868 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:43.702152 3868 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.666523 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0327 10:44:44.722824 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722844 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722851 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722856 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722869 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722874 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722878 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:44.722882 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 10:44:44.844520 3868 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-zwwmh] creating logical port openshift-console_downloads-56d45c498b-zwwmh for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:44.844612 3868 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-zwwmh I0327 10:44:44.862301 3868 pod.go:62] [openshift-console/downloads-56d45c498b-zwwmh] pod update took 17.717851ms I0327 10:44:44.862322 3868 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-zwwmh] addLogicalPort annotation time took 17.743859ms I0327 10:44:44.863488 3868 pods.go:271] [openshift-console/downloads-56d45c498b-zwwmh] addLogicalPort took 18.986943ms, libovsdb time 843.439µs I0327 10:44:45.203174 3868 cni.go:403] [openshift-console/downloads-56d45c498b-zwwmh 740752d335852f7f9784995bea70b3a6a5c9a607cd544a776500f4360b5b2f32 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-zwwmh 740752d335852f7f9784995bea70b3a6a5c9a607cd544a776500f4360b5b2f32 network default NAD default] I0327 10:44:45.212429 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-zwwmh, hostIfaceName: 740752d335852f7, network: default, NAD default, SandboxID: "740752d335852f7f9784995bea70b3a6a5c9a607cd544a776500f4360b5b2f32", PCI device ID: , UID: "4dcb12c8-82fc-45f5-913a-0595e26768ec", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0327 10:44:45.264135 3868 cni.go:424] [openshift-console/downloads-56d45c498b-zwwmh 740752d335852f7f9784995bea70b3a6a5c9a607cd544a776500f4360b5b2f32 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-zwwmh 740752d335852f7f9784995bea70b3a6a5c9a607cd544a776500f4360b5b2f32 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:42:c8:9a:32:58\",\"name\":\"740752d335852f7\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a313d28-1ddd-44ed-ad3e-4a4cc543f5b8\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:48.610866 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:48.610889 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:48.610896 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 10:44:48.610902 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 10:44:52.805141 3868 base_network_controller_pods.go:487] [default/openshift-console/console-5f88f684cc-k96p8] creating logical port openshift-console_console-5f88f684cc-k96p8 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:44:52.805276 3868 kube.go:256] Updating pod openshift-console/console-5f88f684cc-k96p8 I0327 10:44:52.822126 3868 pod.go:62] [openshift-console/console-5f88f684cc-k96p8] pod update took 16.883223ms I0327 10:44:52.822147 3868 base_network_controller_pods.go:937] [default/openshift-console/console-5f88f684cc-k96p8] addLogicalPort annotation time took 16.909305ms I0327 10:44:52.823324 3868 pods.go:271] [openshift-console/console-5f88f684cc-k96p8] addLogicalPort took 18.200121ms, libovsdb time 834.413µs I0327 10:44:53.171158 3868 cni.go:403] [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default] ADD starting CNI request [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default] I0327 10:44:53.180059 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5f88f684cc-k96p8, hostIfaceName: e510120e7569760, network: default, NAD default, SandboxID: "e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7", PCI device ID: , UID: "d0fff7a2-796b-407f-93a5-acc18a2799c1", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0327 10:44:53.230896 3868 cni.go:424] [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default] ADD finished CNI request [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:a7:82:37:77:d4\",\"name\":\"e510120e7569760\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9f90d5d-2080-4709-8ee0-7b90f7da73dc\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:44:56.765200 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 10:45:02.182409 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0327 10:45:02.627682 3868 cni.go:403] [openshift-multus/network-metrics-daemon-85prw 69c57c82308aac1c62fd77742e007b2e8dcf0dd0607c741705c0c7e7915c7dc9 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-85prw 69c57c82308aac1c62fd77742e007b2e8dcf0dd0607c741705c0c7e7915c7dc9 network default NAD default] I0327 10:45:02.636126 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-85prw, hostIfaceName: 69c57c82308aac1, network: default, NAD default, SandboxID: "69c57c82308aac1c62fd77742e007b2e8dcf0dd0607c741705c0c7e7915c7dc9", PCI device ID: , UID: "e6ee9290-e004-46b8-b388-a90c7c9840a5", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0327 10:45:02.685615 3868 cni.go:424] [openshift-multus/network-metrics-daemon-85prw 69c57c82308aac1c62fd77742e007b2e8dcf0dd0607c741705c0c7e7915c7dc9 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-85prw 69c57c82308aac1c62fd77742e007b2e8dcf0dd0607c741705c0c7e7915c7dc9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:b6:1d:8e:fc:55\",\"name\":\"69c57c82308aac1\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/88b829ed-6dab-47c5-aaad-f5524a636ae1\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:45:03.711049 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 10:45:03.711080 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 10:45:03.714401 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 10:45:03.748733 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 10:45:03.806114 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 10:45:03.806136 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 10:45:04.607464 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0327 10:45:05.384289 3868 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d555f8896-77spw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:05.384874 3868 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d555f8896-77spw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 10:45:06.182935 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.182956 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.182961 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.182975 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.182979 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.182983 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.222240 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.222259 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:45:06.222264 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 10:45:06.231349 3868 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d555f8896-77spw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:06.231371 3868 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d555f8896-77spw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:06.239393 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7d555f8896-77spw I0327 10:45:06.239422 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7d555f8896-77spw W0327 10:45:08.717539 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 10:45:08.717550 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 10:45:08.717607 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 10:45:09.728889 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 10:45:12.240744 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 10:45:12.240767 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 10:45:12.286616 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 10:45:12.286638 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 10:45:19.259256 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 10:45:25.294473 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0327 10:45:28.844923 3868 cni.go:403] [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default] DEL starting CNI request [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default] I0327 10:45:28.909510 3868 cni.go:424] [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default] DEL finished CNI request [openshift-console/console-5f88f684cc-k96p8 e510120e75697601a13d104e3c0506b4900a7809283f44028cf939096be49fe7 network default NAD default], result "{}", err I0327 10:45:28.979137 3868 obj_retry.go:464] Detected object openshift-console/console-5f88f684cc-k96p8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:28.979170 3868 pods.go:176] Deleting pod: openshift-console/console-5f88f684cc-k96p8 I0327 10:45:28.980801 3868 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5f88f684cc-k96p8, ips: 10.128.0.24 I0327 10:45:28.980832 3868 obj_retry.go:464] Detected object openshift-console/console-5f88f684cc-k96p8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:28.990782 3868 base_network_controller_pods.go:487] [default/openshift-console/console-58986f9998-4dzjz] creating logical port openshift-console_console-58986f9998-4dzjz for pod on switch ip-10-0-2-4.ec2.internal I0327 10:45:28.990909 3868 kube.go:256] Updating pod openshift-console/console-58986f9998-4dzjz I0327 10:45:29.005092 3868 pod.go:62] [openshift-console/console-58986f9998-4dzjz] pod update took 14.209381ms I0327 10:45:29.005112 3868 base_network_controller_pods.go:937] [default/openshift-console/console-58986f9998-4dzjz] addLogicalPort annotation time took 14.233267ms I0327 10:45:29.006317 3868 pods.go:271] [openshift-console/console-58986f9998-4dzjz] addLogicalPort took 15.551046ms, libovsdb time 834.676µs I0327 10:45:29.407281 3868 cni.go:403] [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default] ADD starting CNI request [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default] I0327 10:45:29.415730 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-58986f9998-4dzjz, hostIfaceName: cfce994bd254ffb, network: default, NAD default, SandboxID: "cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98", PCI device ID: , UID: "84e7d976-82d5-488f-9fd1-b64a50b0bf0f", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0327 10:45:29.460106 3868 cni.go:424] [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default] ADD finished CNI request [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:81:92:9a:ab:d2\",\"name\":\"cfce994bd254ffb\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8462aa72-28b3-431a-8f93-2bd8bee2bdf9\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:45:29.823170 3868 obj_retry.go:458] Detected object openshift-console/console-5f88f684cc-k96p8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:29.823195 3868 obj_retry.go:458] Detected object openshift-console/console-5f88f684cc-k96p8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:29.830860 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f88f684cc-k96p8 I0327 10:45:29.830882 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f88f684cc-k96p8 I0327 10:45:30.464314 3868 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:30.464844 3868 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:30.464875 3868 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:45:30.464890 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 10:45:30.464911 3868 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:45:30.464921 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 10:45:31.492765 3868 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:31.492793 3868 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:31.492809 3868 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:31.492821 3868 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jg8mn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:33.570033 3868 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 10:45:33.581476 3868 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 10:45:37.573264 3868 obj_retry.go:464] Detected object openshift-console/console-5f88f684cc-6gcsg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:37.573860 3868 obj_retry.go:464] Detected object openshift-console/console-5f88f684cc-6gcsg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:37.925345 3868 obj_retry.go:458] Detected object openshift-console/console-5f88f684cc-6gcsg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:37.925368 3868 obj_retry.go:458] Detected object openshift-console/console-5f88f684cc-6gcsg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:37.938153 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f88f684cc-6gcsg I0327 10:45:37.938167 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f88f684cc-6gcsg I0327 10:45:38.397222 3868 obj_retry.go:464] Detected object openshift-ingress/router-default-76b469cc7f-gq7d4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:38.397762 3868 obj_retry.go:464] Detected object openshift-ingress/router-default-76b469cc7f-gq7d4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:39.353624 3868 obj_retry.go:458] Detected object openshift-ingress/router-default-76b469cc7f-gq7d4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:39.353645 3868 obj_retry.go:458] Detected object openshift-ingress/router-default-76b469cc7f-gq7d4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:39.361417 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-76b469cc7f-gq7d4 I0327 10:45:39.361444 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-76b469cc7f-gq7d4 W0327 10:45:39.854011 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 10:45:55.743530 3868 cni.go:403] [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default] I0327 10:45:55.814584 3868 cni.go:424] [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 2a127da69fbe6de153ec58c05fce41895d33120b1c07354a21b0af17db999feb network default NAD default], result "{}", err I0327 10:45:55.877166 3868 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:55.877199 3868 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0327 10:45:55.878464 3868 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.128.0.20 I0327 10:45:55.878491 3868 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:45:56.220365 3868 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:56.220396 3868 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:45:56.228959 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 10:45:56.228984 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 10:45:56.275743 3868 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:45:56.275841 3868 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0327 10:45:56.296612 3868 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 20.812798ms I0327 10:45:56.296632 3868 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 20.836031ms I0327 10:45:56.297980 3868 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 22.272975ms, libovsdb time 909.156µs I0327 10:45:56.634817 3868 cni.go:403] [openshift-monitoring/alertmanager-main-1 b99730af7ad2c9af5b95cfc827f5f1b5e8d03b79200779480ce692d6cb816036 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 b99730af7ad2c9af5b95cfc827f5f1b5e8d03b79200779480ce692d6cb816036 network default NAD default] I0327 10:45:56.648702 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: b99730af7ad2c9a, network: default, NAD default, SandboxID: "b99730af7ad2c9af5b95cfc827f5f1b5e8d03b79200779480ce692d6cb816036", PCI device ID: , UID: "ff822b3c-9d16-4bdc-a87e-1e00cb83eb1e", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0327 10:45:56.696074 3868 cni.go:424] [openshift-monitoring/alertmanager-main-1 b99730af7ad2c9af5b95cfc827f5f1b5e8d03b79200779480ce692d6cb816036 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 b99730af7ad2c9af5b95cfc827f5f1b5e8d03b79200779480ce692d6cb816036 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:67:48:09:f3:a7\",\"name\":\"b99730af7ad2c9a\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c81419f-be7e-4afd-8d1d-717e0c2bca7d\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:46:00.838621 3868 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:00.839245 3868 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:01.464491 3868 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:01.464517 3868 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:01.472960 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 10:46:01.472975 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0327 10:46:16.694350 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:46:16.694370 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 10:46:16.694375 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 10:46:17.995099 3868 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:17.995988 3868 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:18.394813 3868 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:18.394840 3868 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:18.405766 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 10:46:18.405791 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 10:46:28.028655 3868 cni.go:403] [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default] DEL starting CNI request [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default] I0327 10:46:28.097834 3868 cni.go:424] [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default] DEL finished CNI request [openshift-console/console-58986f9998-4dzjz cfce994bd254ffbbf472898ed163e0cc95103ebbe1a513f9bdacbb94e3b67b98 network default NAD default], result "{}", err I0327 10:46:28.147889 3868 obj_retry.go:464] Detected object openshift-console/console-58986f9998-4dzjz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:28.147930 3868 pods.go:176] Deleting pod: openshift-console/console-58986f9998-4dzjz I0327 10:46:28.149271 3868 pods.go:236] Attempting to release IPs for pod: openshift-console/console-58986f9998-4dzjz, ips: 10.128.0.25 I0327 10:46:28.149302 3868 obj_retry.go:464] Detected object openshift-console/console-58986f9998-4dzjz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:28.159956 3868 base_network_controller_pods.go:487] [default/openshift-console/console-74b78f9445-ndnx7] creating logical port openshift-console_console-74b78f9445-ndnx7 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:46:28.160048 3868 kube.go:256] Updating pod openshift-console/console-74b78f9445-ndnx7 I0327 10:46:28.174822 3868 pod.go:62] [openshift-console/console-74b78f9445-ndnx7] pod update took 14.798878ms I0327 10:46:28.174845 3868 base_network_controller_pods.go:937] [default/openshift-console/console-74b78f9445-ndnx7] addLogicalPort annotation time took 14.827668ms I0327 10:46:28.176069 3868 pods.go:271] [openshift-console/console-74b78f9445-ndnx7] addLogicalPort took 16.122506ms, libovsdb time 901.322µs I0327 10:46:28.822491 3868 cni.go:403] [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default] ADD starting CNI request [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default] I0327 10:46:28.831016 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-74b78f9445-ndnx7, hostIfaceName: cda20133680c74c, network: default, NAD default, SandboxID: "cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f", PCI device ID: , UID: "59f5f3e5-d32a-435a-9b42-b46cacdbad70", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0327 10:46:28.875501 3868 cni.go:424] [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default] ADD finished CNI request [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:b4:42:e5:0b:f4\",\"name\":\"cda20133680c74c\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24de64b1-f67a-4e82-bbf6-9f0f18b4e30d\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:46:29.096660 3868 obj_retry.go:458] Detected object openshift-console/console-58986f9998-4dzjz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:29.096681 3868 obj_retry.go:458] Detected object openshift-console/console-58986f9998-4dzjz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:29.105181 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-58986f9998-4dzjz I0327 10:46:29.105206 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-58986f9998-4dzjz W0327 10:46:39.148388 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 10:46:39.399464 3868 obj_retry.go:464] Detected object openshift-console/console-58986f9998-rtjq6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:39.400003 3868 obj_retry.go:464] Detected object openshift-console/console-58986f9998-rtjq6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:46:39.610770 3868 obj_retry.go:458] Detected object openshift-console/console-58986f9998-rtjq6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:39.610792 3868 obj_retry.go:458] Detected object openshift-console/console-58986f9998-rtjq6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:46:39.619690 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-58986f9998-rtjq6 I0327 10:46:39.619725 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-58986f9998-rtjq6 I0327 10:47:04.051030 3868 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:47:04.051527 3868 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:47:04.296939 3868 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:47:04.296964 3868 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:47:04.305534 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 10:47:04.305560 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 10:47:24.309738 3868 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 10:47:30.349311 3868 egressfirewall.go:628] Syncing node "ip-10-0-0-12.ec2.internal" for egress firewall I0327 10:47:30.349325 3868 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-12.ec2.internal I0327 10:47:30.349347 3868 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-12.ec2.internal: 28.354µs I0327 10:48:19.620811 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=634 I0327 10:48:27.708087 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0327 10:49:04.712626 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0327 10:49:10.038352 3868 cni.go:403] [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default] DEL starting CNI request [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default] I0327 10:49:10.106291 3868 cni.go:424] [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default] DEL finished CNI request [openshift-console/console-74b78f9445-ndnx7 cda20133680c74c3b925f76a5f1a93bb2665f4de1f3da63c1631b7e5b72dff3f network default NAD default], result "{}", err I0327 10:49:10.158698 3868 obj_retry.go:464] Detected object openshift-console/console-74b78f9445-ndnx7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:49:10.158752 3868 pods.go:176] Deleting pod: openshift-console/console-74b78f9445-ndnx7 I0327 10:49:10.159960 3868 pods.go:236] Attempting to release IPs for pod: openshift-console/console-74b78f9445-ndnx7, ips: 10.128.0.27 I0327 10:49:10.159988 3868 obj_retry.go:464] Detected object openshift-console/console-74b78f9445-ndnx7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:49:10.168117 3868 base_network_controller_pods.go:487] [default/openshift-console/console-7fdc5b6d67-qdrlf] creating logical port openshift-console_console-7fdc5b6d67-qdrlf for pod on switch ip-10-0-2-4.ec2.internal I0327 10:49:10.168227 3868 kube.go:256] Updating pod openshift-console/console-7fdc5b6d67-qdrlf I0327 10:49:10.184217 3868 pod.go:62] [openshift-console/console-7fdc5b6d67-qdrlf] pod update took 16.024425ms I0327 10:49:10.184242 3868 base_network_controller_pods.go:937] [default/openshift-console/console-7fdc5b6d67-qdrlf] addLogicalPort annotation time took 16.053656ms I0327 10:49:10.185435 3868 pods.go:271] [openshift-console/console-7fdc5b6d67-qdrlf] addLogicalPort took 17.330575ms, libovsdb time 826.363µs I0327 10:49:10.525557 3868 cni.go:403] [openshift-console/console-7fdc5b6d67-qdrlf e60001aacc9881a0203ff2d73ab3e77c859e857f3ea2dabcda7a4c28d553b5fe network default NAD default] ADD starting CNI request [openshift-console/console-7fdc5b6d67-qdrlf e60001aacc9881a0203ff2d73ab3e77c859e857f3ea2dabcda7a4c28d553b5fe network default NAD default] I0327 10:49:10.534255 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7fdc5b6d67-qdrlf, hostIfaceName: e60001aacc9881a, network: default, NAD default, SandboxID: "e60001aacc9881a0203ff2d73ab3e77c859e857f3ea2dabcda7a4c28d553b5fe", PCI device ID: , UID: "c3dc5744-5a7f-4582-9464-adf80284aceb", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0327 10:49:10.578643 3868 cni.go:424] [openshift-console/console-7fdc5b6d67-qdrlf e60001aacc9881a0203ff2d73ab3e77c859e857f3ea2dabcda7a4c28d553b5fe network default NAD default] ADD finished CNI request [openshift-console/console-7fdc5b6d67-qdrlf e60001aacc9881a0203ff2d73ab3e77c859e857f3ea2dabcda7a4c28d553b5fe network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:32:bb:3c:fc:69\",\"name\":\"e60001aacc9881a\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fd823ad4-7591-4d7b-a6f0-a736781983cf\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:49:10.750974 3868 obj_retry.go:458] Detected object openshift-console/console-74b78f9445-ndnx7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:49:10.750999 3868 obj_retry.go:458] Detected object openshift-console/console-74b78f9445-ndnx7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:49:10.757897 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-74b78f9445-ndnx7 I0327 10:49:10.757924 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-74b78f9445-ndnx7 I0327 10:49:15.610562 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=140 W0327 10:49:20.782350 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 10:49:21.473527 3868 obj_retry.go:464] Detected object openshift-console/console-74b78f9445-v2w9t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:49:21.474054 3868 obj_retry.go:464] Detected object openshift-console/console-74b78f9445-v2w9t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:49:21.848794 3868 obj_retry.go:458] Detected object openshift-console/console-74b78f9445-v2w9t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:49:21.848817 3868 obj_retry.go:458] Detected object openshift-console/console-74b78f9445-v2w9t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:49:21.856331 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-74b78f9445-v2w9t I0327 10:49:21.856354 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-74b78f9445-v2w9t I0327 10:50:00.204172 3868 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576810-mckv5 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:50:00.204269 3868 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 I0327 10:50:00.218286 3868 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5] pod update took 14.046928ms I0327 10:50:00.218307 3868 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5] addLogicalPort annotation time took 14.072157ms I0327 10:50:00.219509 3868 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5] addLogicalPort took 15.357771ms, libovsdb time 870.074µs I0327 10:50:00.586293 3868 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default] I0327 10:50:00.594943 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576810-mckv5, hostIfaceName: bb4282fa5850a6e, network: default, NAD default, SandboxID: "bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31", PCI device ID: , UID: "1dbd8e2f-8630-4b42-808f-65ea0c6b7b5b", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0327 10:50:00.639750 3868 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:ac:b2:ad:aa:fd\",\"name\":\"bb4282fa5850a6e\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4bd668c6-6e35-4cbc-9644-b51c94c5163d\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:50:04.964621 3868 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default] I0327 10:50:05.041100 3868 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 bb4282fa5850a6e9a3b858ffd0f27716f7b8f98690e036e2d02f1e112a767e31 network default NAD default], result "{}", err I0327 10:50:05.084890 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:50:05.084929 3868 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 I0327 10:50:05.086316 3868 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5, ips: 10.128.0.29 I0327 10:50:05.086346 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:50:05.982755 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:50:05.982780 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:50:35.427016 3868 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-s5xhr for pod on switch ip-10-0-2-4.ec2.internal I0327 10:50:35.427147 3868 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr I0327 10:50:35.446427 3868 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr] pod update took 19.307413ms I0327 10:50:35.446461 3868 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr] addLogicalPort annotation time took 19.344326ms I0327 10:50:35.448050 3868 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr] addLogicalPort took 21.050592ms, libovsdb time 1.121573ms I0327 10:50:35.785465 3868 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr 6628cc8fe943e9d2c8d3b4a7dd9c73cc5d8997ec9b133f5c2f841f3c77c3534c network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr 6628cc8fe943e9d2c8d3b4a7dd9c73cc5d8997ec9b133f5c2f841f3c77c3534c network default NAD default] I0327 10:50:35.794448 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-s5xhr, hostIfaceName: 6628cc8fe943e9d, network: default, NAD default, SandboxID: "6628cc8fe943e9d2c8d3b4a7dd9c73cc5d8997ec9b133f5c2f841f3c77c3534c", PCI device ID: , UID: "8972504c-32c1-42f3-b203-10df58020b7d", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0327 10:50:35.839229 3868 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr 6628cc8fe943e9d2c8d3b4a7dd9c73cc5d8997ec9b133f5c2f841f3c77c3534c network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-s5xhr 6628cc8fe943e9d2c8d3b4a7dd9c73cc5d8997ec9b133f5c2f841f3c77c3534c network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:74:ff:2f:b9:9a\",\"name\":\"6628cc8fe943e9d\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8b54e825-9525-4ce2-8ee9-5e25052c1828\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:50:39.101388 3868 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 10:50:44.739941 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 10:50:49.699564 3868 namespace.go:98] [tsf] adding namespace I0327 10:50:49.700440 3868 namespace.go:102] [tsf] adding namespace took 854.14µs I0327 10:50:49.747501 3868 namespace.go:142] [tsf] updating namespace I0327 10:50:49.774086 3868 namespace.go:142] [tsf] updating namespace W0327 10:50:50.118001 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 10:50:50.156017 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 10:50:50.156084 3868 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-m26ks IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-m26ks: no pod IPs found I0327 10:50:53.457081 3868 namespace.go:98] [cert-manager-operator] adding namespace I0327 10:50:53.457945 3868 namespace.go:102] [cert-manager-operator] adding namespace took 842.484µs I0327 10:50:53.459997 3868 namespace.go:98] [tsf-keycloak] adding namespace I0327 10:50:53.460761 3868 namespace.go:102] [tsf-keycloak] adding namespace took 750.269µs I0327 10:50:53.473284 3868 namespace.go:98] [konflux-operator] adding namespace I0327 10:50:53.473333 3868 namespace.go:98] [konflux-ui] adding namespace I0327 10:50:53.473565 3868 namespace.go:98] [tsf-tpa] adding namespace I0327 10:50:53.473638 3868 namespace.go:98] [rhtpa-operator] adding namespace I0327 10:50:53.473734 3868 namespace.go:98] [rhbk-operator] adding namespace I0327 10:50:53.474071 3868 namespace.go:102] [konflux-operator] adding namespace took 767.077µs I0327 10:50:53.474675 3868 namespace.go:102] [konflux-ui] adding namespace took 1.329479ms I0327 10:50:53.474702 3868 namespace.go:98] [tsf-tas] adding namespace I0327 10:50:53.475286 3868 namespace.go:102] [tsf-tas] adding namespace took 578.546µs I0327 10:50:53.475744 3868 namespace.go:102] [tsf-tpa] adding namespace took 2.162158ms I0327 10:50:53.476353 3868 namespace.go:102] [rhtpa-operator] adding namespace took 2.698469ms I0327 10:50:53.476871 3868 namespace.go:102] [rhbk-operator] adding namespace took 3.098167ms I0327 10:50:53.490912 3868 namespace.go:142] [cert-manager-operator] updating namespace I0327 10:50:53.508054 3868 namespace.go:142] [cert-manager-operator] updating namespace I0327 10:50:53.512653 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:50:53.512827 3868 namespace.go:142] [konflux-operator] updating namespace I0327 10:50:53.512894 3868 namespace.go:142] [tsf-tas] updating namespace I0327 10:50:53.517548 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:50:53.523332 3868 namespace.go:142] [rhtpa-operator] updating namespace I0327 10:50:53.524380 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:50:53.524460 3868 namespace.go:142] [rhbk-operator] updating namespace I0327 10:50:53.534092 3868 namespace.go:142] [cert-manager-operator] updating namespace I0327 10:50:53.569220 3868 namespace.go:142] [konflux-operator] updating namespace I0327 10:50:53.587489 3868 namespace.go:142] [konflux-operator] updating namespace I0327 10:50:53.599659 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:50:53.631880 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:50:53.639134 3868 namespace.go:142] [tsf-tas] updating namespace I0327 10:50:53.656052 3868 namespace.go:142] [tsf-tas] updating namespace I0327 10:50:53.663306 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:50:53.686071 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:50:53.699906 3868 namespace.go:142] [rhtpa-operator] updating namespace I0327 10:50:53.716735 3868 namespace.go:142] [rhtpa-operator] updating namespace I0327 10:50:53.735730 3868 namespace.go:142] [rhbk-operator] updating namespace I0327 10:50:53.741857 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0327 10:50:53.748300 3868 namespace.go:142] [rhbk-operator] updating namespace I0327 10:50:53.766407 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:50:53.788978 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:50:54.652184 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:50:54.668000 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:51:00.752973 3868 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:00.753481 3868 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:00.753508 3868 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:00.753520 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 10:51:00.753535 3868 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:00.753540 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 10:51:01.768114 3868 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:01.768674 3868 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:01.768728 3868 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:01.768742 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 10:51:01.768754 3868 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:01.768758 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 10:51:01.784405 3868 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:01.784423 3868 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:01.784436 3868 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:01.784445 3868 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8plccb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:02.796864 3868 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:02.796882 3868 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:02.796892 3868 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:02.796901 3868 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12hrdk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:05.819387 3868 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.819940 3868 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.819961 3868 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.819968 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 10:51:05.819977 3868 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.819982 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 10:51:05.931127 3868 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.931600 3868 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.931635 3868 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.931647 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 10:51:05.931659 3868 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:05.931667 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 10:51:06.805041 3868 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:06.805568 3868 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:06.805598 3868 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:06.805608 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 10:51:06.805621 3868 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:06.805629 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 10:51:06.851879 3868 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.851899 3868 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.851912 3868 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.851921 3868 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8nwf7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.963589 3868 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.963605 3868 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.963625 3868 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:06.963638 3868 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5fb4qs of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:07.807128 3868 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:07.807668 3868 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:51:07.807700 3868 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:07.807731 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 10:51:07.807747 3868 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 10:51:07.807753 3868 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 10:51:07.840347 3868 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:07.840366 3868 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:07.840382 3868 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:07.840392 3868 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7drffv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:08.834682 3868 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:08.834704 3868 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:08.834743 3868 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:08.834758 3868 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafljs74 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:51:11.072961 3868 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-jvjb9 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:51:11.073071 3868 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9 I0327 10:51:11.087943 3868 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9] pod update took 14.897692ms I0327 10:51:11.087969 3868 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9] addLogicalPort annotation time took 14.927164ms I0327 10:51:11.089551 3868 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9] addLogicalPort took 16.60536ms, libovsdb time 1.009099ms I0327 10:51:11.437319 3868 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9 19af5106768dd7737299463ab3df02cc3be6d6d398fdbff7ed59bc0fa1604075 network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9 19af5106768dd7737299463ab3df02cc3be6d6d398fdbff7ed59bc0fa1604075 network default NAD default] I0327 10:51:11.446207 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-jvjb9, hostIfaceName: 19af5106768dd77, network: default, NAD default, SandboxID: "19af5106768dd7737299463ab3df02cc3be6d6d398fdbff7ed59bc0fa1604075", PCI device ID: , UID: "03bc714b-f52e-41cd-a8b1-2da39e66742e", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0327 10:51:11.491682 3868 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9 19af5106768dd7737299463ab3df02cc3be6d6d398fdbff7ed59bc0fa1604075 network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-jvjb9 19af5106768dd7737299463ab3df02cc3be6d6d398fdbff7ed59bc0fa1604075 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:25:8d:4f:6b:16\",\"name\":\"19af5106768dd77\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5d32417-8bf7-414e-9918-305da0573350\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:51:14.733908 3868 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 W0327 10:51:15.258446 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 10:51:16.977066 3868 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-c7c9x for pod on switch ip-10-0-2-4.ec2.internal I0327 10:51:16.977180 3868 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x I0327 10:51:16.996179 3868 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x] pod update took 19.027323ms I0327 10:51:16.996199 3868 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x] addLogicalPort annotation time took 19.051695ms I0327 10:51:16.997405 3868 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x] addLogicalPort took 20.359793ms, libovsdb time 854.734µs I0327 10:51:17.335485 3868 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x 1dcd7d2a1f3574af51fba19e39c0a7f14e3d2edcb5cbcf8b93156e6c2e075135 network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x 1dcd7d2a1f3574af51fba19e39c0a7f14e3d2edcb5cbcf8b93156e6c2e075135 network default NAD default] I0327 10:51:17.343984 3868 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-c7c9x, hostIfaceName: 1dcd7d2a1f3574a, network: default, NAD default, SandboxID: "1dcd7d2a1f3574af51fba19e39c0a7f14e3d2edcb5cbcf8b93156e6c2e075135", PCI device ID: , UID: "287c25d8-5abe-47e9-99b9-fde1df9a5184", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0327 10:51:17.389231 3868 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x 1dcd7d2a1f3574af51fba19e39c0a7f14e3d2edcb5cbcf8b93156e6c2e075135 network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-c7c9x 1dcd7d2a1f3574af51fba19e39c0a7f14e3d2edcb5cbcf8b93156e6c2e075135 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:97:b7:cc:a8:e9\",\"name\":\"1dcd7d2a1f3574a\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/132c467b-9097-4189-9769-691ad5bec00f\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:51:19.725834 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0327 10:51:20.525842 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0327 10:51:20.540427 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 10:51:21.357637 3868 namespace.go:142] [openshift-operators] updating namespace I0327 10:51:21.373037 3868 namespace.go:142] [openshift-operators] updating namespace I0327 10:51:22.546072 3868 namespace.go:98] [cert-manager] adding namespace I0327 10:51:22.546990 3868 namespace.go:102] [cert-manager] adding namespace took 888.335µs I0327 10:51:22.575081 3868 namespace.go:142] [cert-manager] updating namespace I0327 10:51:22.599945 3868 namespace.go:142] [cert-manager] updating namespace I0327 10:51:24.398154 3868 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-xz6nz for pod on switch ip-10-0-2-4.ec2.internal I0327 10:51:24.398246 3868 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz I0327 10:51:24.424629 3868 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz] pod update took 26.404927ms I0327 10:51:24.424652 3868 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz] addLogicalPort annotation time took 26.43236ms I0327 10:51:24.425851 3868 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz] addLogicalPort took 27.710823ms, libovsdb time 819.568µs I0327 10:51:24.770025 3868 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz dfe27964fccfe2dab801238bd178693c7b84cc535cf9dfdac7e8e4910c67dab9 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz dfe27964fccfe2dab801238bd178693c7b84cc535cf9dfdac7e8e4910c67dab9 network default NAD default] I0327 10:51:24.778905 3868 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-xz6nz, hostIfaceName: dfe27964fccfe2d, network: default, NAD default, SandboxID: "dfe27964fccfe2dab801238bd178693c7b84cc535cf9dfdac7e8e4910c67dab9", PCI device ID: , UID: "5a4ed058-157d-4f5a-a986-d4aa687a3363", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0327 10:51:24.823660 3868 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz dfe27964fccfe2dab801238bd178693c7b84cc535cf9dfdac7e8e4910c67dab9 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-xz6nz dfe27964fccfe2dab801238bd178693c7b84cc535cf9dfdac7e8e4910c67dab9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:88:93:c4:17:33\",\"name\":\"dfe27964fccfe2d\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9097735-6d46-41aa-9248-0ba4b6f1900b\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:51:26.330084 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 10:51:26.915645 3868 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm] creating logical port cert-manager_cert-manager-cainjector-5b4bf68748-z9tvm for pod on switch ip-10-0-2-4.ec2.internal I0327 10:51:26.915791 3868 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm I0327 10:51:26.957066 3868 pod.go:62] [cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm] pod update took 41.312823ms I0327 10:51:26.957097 3868 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm] addLogicalPort annotation time took 41.349728ms I0327 10:51:26.958305 3868 pods.go:271] [cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm] addLogicalPort took 42.690276ms, libovsdb time 794.006µs I0327 10:51:27.603292 3868 cni.go:403] [cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm 18985dd0ead2ec8a080173a4054bc5e33e25d6e0f39377282b04901517096c72 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm 18985dd0ead2ec8a080173a4054bc5e33e25d6e0f39377282b04901517096c72 network default NAD default] I0327 10:51:27.613839 3868 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5b4bf68748-z9tvm, hostIfaceName: 18985dd0ead2ec8, network: default, NAD default, SandboxID: "18985dd0ead2ec8a080173a4054bc5e33e25d6e0f39377282b04901517096c72", PCI device ID: , UID: "128c426f-4d0f-486a-a43b-5f001a87ce96", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0327 10:51:27.660034 3868 cni.go:424] [cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm 18985dd0ead2ec8a080173a4054bc5e33e25d6e0f39377282b04901517096c72 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-z9tvm 18985dd0ead2ec8a080173a4054bc5e33e25d6e0f39377282b04901517096c72 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:8e:4a:78:f9:fb\",\"name\":\"18985dd0ead2ec8\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05c6ff3f-ba0f-4401-b81f-df51aaf21654\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:51:28.358732 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 10:51:28.358750 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 10:51:31.125941 3868 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 10:51:31.126754 3868 namespace.go:102] [trusted-artifact-signer] adding namespace took 788.061µs I0327 10:51:31.162967 3868 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 10:51:31.182672 3868 namespace.go:142] [trusted-artifact-signer] updating namespace W0327 10:51:31.865988 3868 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 10:51:32.205282 3868 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-lxkwj for pod on switch ip-10-0-2-4.ec2.internal I0327 10:51:32.205367 3868 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj I0327 10:51:32.244249 3868 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj] pod update took 38.903395ms I0327 10:51:32.244271 3868 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj] addLogicalPort annotation time took 38.930259ms I0327 10:51:32.245441 3868 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj] addLogicalPort took 40.173307ms, libovsdb time 832.818µs I0327 10:51:32.564105 3868 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj f53b1664cacfe86d8abffb8556d0e13331e6d512873f611d4c03610ff62021e1 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj f53b1664cacfe86d8abffb8556d0e13331e6d512873f611d4c03610ff62021e1 network default NAD default] I0327 10:51:32.586992 3868 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-lxkwj, hostIfaceName: f53b1664cacfe86, network: default, NAD default, SandboxID: "f53b1664cacfe86d8abffb8556d0e13331e6d512873f611d4c03610ff62021e1", PCI device ID: , UID: "4cf79f01-4524-4689-9d8d-4ae713116312", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0327 10:51:32.620861 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=61 I0327 10:51:32.633152 3868 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj f53b1664cacfe86d8abffb8556d0e13331e6d512873f611d4c03610ff62021e1 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-lxkwj f53b1664cacfe86d8abffb8556d0e13331e6d512873f611d4c03610ff62021e1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:38:a7:56:00:2f\",\"name\":\"f53b1664cacfe86\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/95be72c1-807b-4fd8-8d93-76a3e889546b\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:51:32.850502 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 10:51:34.352147 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 10:51:34.352176 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 10:51:35.095028 3868 namespace.go:142] [openshift-operators] updating namespace I0327 10:51:36.205240 3868 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-dn8sz for pod on switch ip-10-0-2-4.ec2.internal I0327 10:51:36.205367 3868 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz I0327 10:51:36.252686 3868 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz] pod update took 47.349479ms I0327 10:51:36.252728 3868 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz] addLogicalPort annotation time took 47.381251ms I0327 10:51:36.254152 3868 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz] addLogicalPort took 48.930223ms, libovsdb time 1.031501ms I0327 10:51:36.556555 3868 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz 13294ec089696d0daa4be7b220dd48abd7901d21883068133b804b29fb358812 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz 13294ec089696d0daa4be7b220dd48abd7901d21883068133b804b29fb358812 network default NAD default] I0327 10:51:36.565265 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-dn8sz, hostIfaceName: 13294ec089696d0, network: default, NAD default, SandboxID: "13294ec089696d0daa4be7b220dd48abd7901d21883068133b804b29fb358812", PCI device ID: , UID: "0fab32fd-ebb3-4ea0-8d10-9924be1bbe96", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0327 10:51:36.615108 3868 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz 13294ec089696d0daa4be7b220dd48abd7901d21883068133b804b29fb358812 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-dn8sz 13294ec089696d0daa4be7b220dd48abd7901d21883068133b804b29fb358812 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:84:56:18:d1:6b\",\"name\":\"13294ec089696d0\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bdcfa562-152c-4b07-9b5a-bb1edd14b99e\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:51:49.704911 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0327 10:51:52.615177 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0327 10:51:54.046165 3868 namespace.go:98] [openshift-pipelines] adding namespace I0327 10:51:54.047094 3868 namespace.go:102] [openshift-pipelines] adding namespace took 907.45µs I0327 10:51:54.070448 3868 namespace.go:142] [openshift-pipelines] updating namespace I0327 10:51:54.752274 3868 namespace.go:142] [cert-manager] updating namespace I0327 10:51:54.957343 3868 namespace.go:142] [cert-manager-operator] updating namespace I0327 10:51:55.356737 3868 namespace.go:142] [konflux-operator] updating namespace I0327 10:51:55.558983 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:51:55.760578 3868 namespace.go:142] [open-cluster-management-2pa0ron8lulpu1e8444fbruk00uofggt] updating namespace I0327 10:51:56.160805 3868 namespace.go:142] [rhbk-operator] updating namespace I0327 10:51:56.363180 3868 namespace.go:142] [rhtpa-operator] updating namespace I0327 10:51:56.562078 3868 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 10:51:56.765199 3868 namespace.go:142] [tsf] updating namespace I0327 10:51:56.966625 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:51:57.168009 3868 namespace.go:142] [tsf-tas] updating namespace I0327 10:51:57.365159 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:51:57.500613 3868 namespace.go:142] [cert-manager] updating namespace I0327 10:51:57.509454 3868 namespace.go:142] [cert-manager-operator] updating namespace I0327 10:51:57.520571 3868 namespace.go:142] [default] updating namespace I0327 10:51:57.530500 3868 namespace.go:142] [konflux-operator] updating namespace I0327 10:51:57.540738 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:51:57.553378 3868 namespace.go:142] [open-cluster-management-2pa0ron8lulpu1e8444fbruk00uofggt] updating namespace I0327 10:51:57.562016 3868 namespace.go:142] [openshift] updating namespace I0327 10:51:57.572279 3868 namespace.go:142] [rhbk-operator] updating namespace I0327 10:51:57.583857 3868 namespace.go:142] [rhtpa-operator] updating namespace I0327 10:51:57.597875 3868 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 10:51:57.608998 3868 namespace.go:142] [tsf] updating namespace I0327 10:51:57.619860 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:51:57.628575 3868 namespace.go:142] [tsf-tas] updating namespace I0327 10:51:57.650858 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:51:57.748420 3868 namespace.go:142] [cert-manager] updating namespace I0327 10:51:57.848050 3868 namespace.go:142] [cert-manager-operator] updating namespace I0327 10:51:57.948058 3868 namespace.go:142] [default] updating namespace I0327 10:51:58.050133 3868 namespace.go:142] [konflux-operator] updating namespace I0327 10:51:58.147960 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:51:58.254266 3868 namespace.go:142] [open-cluster-management-2pa0ron8lulpu1e8444fbruk00uofggt] updating namespace I0327 10:51:58.346381 3868 namespace.go:142] [openshift] updating namespace I0327 10:51:58.447934 3868 namespace.go:142] [rhbk-operator] updating namespace I0327 10:51:58.551941 3868 namespace.go:142] [rhtpa-operator] updating namespace I0327 10:51:58.649153 3868 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 10:51:58.748137 3868 namespace.go:142] [tsf] updating namespace I0327 10:51:58.849457 3868 namespace.go:142] [tsf-keycloak] updating namespace I0327 10:51:58.950230 3868 namespace.go:142] [tsf-tas] updating namespace I0327 10:51:59.049954 3868 namespace.go:142] [tsf-tpa] updating namespace I0327 10:51:59.347992 3868 namespace.go:142] [openshift-pipelines] updating namespace I0327 10:52:07.149596 3868 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-jjn8h for pod on switch ip-10-0-2-4.ec2.internal I0327 10:52:07.149746 3868 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h I0327 10:52:07.163078 3868 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h] pod update took 13.372968ms I0327 10:52:07.163097 3868 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h] addLogicalPort annotation time took 13.398321ms I0327 10:52:07.164375 3868 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h] addLogicalPort took 14.796284ms, libovsdb time 860.959µs I0327 10:52:07.527726 3868 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h 9046c0fa841edbd3c7b161676706b868d7b33038425ef0f3f5876dd974b74897 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h 9046c0fa841edbd3c7b161676706b868d7b33038425ef0f3f5876dd974b74897 network default NAD default] I0327 10:52:07.537329 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-jjn8h, hostIfaceName: 9046c0fa841edbd, network: default, NAD default, SandboxID: "9046c0fa841edbd3c7b161676706b868d7b33038425ef0f3f5876dd974b74897", PCI device ID: , UID: "c9cf5af4-a02a-456b-b77e-8b3c5b4296c3", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0327 10:52:07.587502 3868 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h 9046c0fa841edbd3c7b161676706b868d7b33038425ef0f3f5876dd974b74897 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-jjn8h 9046c0fa841edbd3c7b161676706b868d7b33038425ef0f3f5876dd974b74897 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:d2:07:3b:80:4b\",\"name\":\"9046c0fa841edbd\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0cbedd67-0af0-49af-8685-61ca0e8d13d9\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:52:07.680312 3868 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:52:07.680931 3868 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:52:09.703266 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0327 10:52:11.035417 3868 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 10:52:11.035436 3868 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 10:52:11.035441 3868 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 10:52:11.035444 3868 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 10:52:11.059641 3868 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 10:52:11.059655 3868 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 10:52:11.059661 3868 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 10:52:11.091421 3868 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 10:52:11.091433 3868 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 10:52:11.091437 3868 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0327 10:52:15.612551 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=96 I0327 10:52:15.729137 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0327 10:52:16.034119 3868 namespace.go:142] [openshift-pipelines] updating namespace W0327 10:52:33.736219 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 10:52:46.718674 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0327 10:52:52.726180 3868 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 10:53:04.614427 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=101 W0327 10:53:16.900768 3868 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 10:53:16.900784 3868 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 10:53:16.900789 3868 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 10:53:46.733125 3868 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:53:46.733153 3868 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:53:46.741272 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 10:53:46.741297 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 10:53:50.858074 3868 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-s4n28 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:53:50.858222 3868 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28 I0327 10:53:50.873734 3868 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28] pod update took 15.587205ms I0327 10:53:50.873761 3868 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28] addLogicalPort annotation time took 15.619196ms I0327 10:53:50.875127 3868 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28] addLogicalPort took 17.070595ms, libovsdb time 835.756µs I0327 10:53:52.828980 3868 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-r2nfd for pod on switch ip-10-0-2-4.ec2.internal I0327 10:53:52.829094 3868 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd I0327 10:53:52.842566 3868 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd] pod update took 13.49531ms I0327 10:53:52.842586 3868 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd] addLogicalPort annotation time took 13.52062ms I0327 10:53:52.843932 3868 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd] addLogicalPort took 14.966339ms, libovsdb time 894.857µs I0327 10:53:53.189328 3868 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd 39af30f547569dbd2b2ebcc9b8fe3badd7d6f140fdb53319b7f2b31d24045cb4 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd 39af30f547569dbd2b2ebcc9b8fe3badd7d6f140fdb53319b7f2b31d24045cb4 network default NAD default] I0327 10:53:53.197934 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-r2nfd, hostIfaceName: 39af30f547569db, network: default, NAD default, SandboxID: "39af30f547569dbd2b2ebcc9b8fe3badd7d6f140fdb53319b7f2b31d24045cb4", PCI device ID: , UID: "51a6cec5-3b4e-4cd1-aab9-161bfff5c0cf", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0327 10:53:53.244489 3868 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd 39af30f547569dbd2b2ebcc9b8fe3badd7d6f140fdb53319b7f2b31d24045cb4 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-r2nfd 39af30f547569dbd2b2ebcc9b8fe3badd7d6f140fdb53319b7f2b31d24045cb4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:c6:6b:9b:c9:d4\",\"name\":\"39af30f547569db\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05de2c0f-a9d1-4c60-9ee1-d519cf4b322f\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:53:56.034240 3868 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 10:53:56.034258 3868 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 10:53:56.064320 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 10:53:56.064339 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 10:53:58.122852 3868 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 10:53:58.130760 3868 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28 a0b53d7dc70b45801a54d27b02814657a06440cf0fe6469b2313e29ba5063f06 network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28 a0b53d7dc70b45801a54d27b02814657a06440cf0fe6469b2313e29ba5063f06 network default NAD default] I0327 10:53:58.139242 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-s4n28, hostIfaceName: a0b53d7dc70b458, network: default, NAD default, SandboxID: "a0b53d7dc70b45801a54d27b02814657a06440cf0fe6469b2313e29ba5063f06", PCI device ID: , UID: "2e8588c5-84dc-4670-bb90-ffb536ce9023", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0327 10:53:58.186776 3868 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28 a0b53d7dc70b45801a54d27b02814657a06440cf0fe6469b2313e29ba5063f06 network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-s4n28 a0b53d7dc70b45801a54d27b02814657a06440cf0fe6469b2313e29ba5063f06 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:21:5e:d2:59:c4\",\"name\":\"a0b53d7dc70b458\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/372fd68f-22ff-4c77-b769-31220768b277\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:54:00.599994 3868 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 10:54:03.075882 3868 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0327 10:54:07.623814 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=328 W0327 10:54:24.083257 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0327 10:54:27.460280 3868 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:54:27.460839 3868 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:54:27.480866 3868 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:54:27.480888 3868 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:54:27.488078 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 10:54:27.488101 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 10:54:28.562738 3868 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:54:28.562847 3868 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 10:54:28.582143 3868 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 19.328892ms I0327 10:54:28.582165 3868 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 19.356988ms I0327 10:54:28.583375 3868 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 20.679173ms, libovsdb time 844.232µs I0327 10:54:28.926246 3868 cni.go:403] [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default] I0327 10:54:28.935229 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 4cbaa3a9942176c, network: default, NAD default, SandboxID: "4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb", PCI device ID: , UID: "e841a864-ab39-4c58-9983-d635981c70db", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0327 10:54:28.983548 3868 cni.go:424] [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:6e:a9:92:a0:ed\",\"name\":\"4cbaa3a9942176c\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a14f8a00-5f91-41b6-8f91-45d0391fd9d8\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:54:36.240659 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:54:36.240680 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:54:36.244139 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 10:54:56.224912 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:54:56.224934 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:54:56.225813 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 10:54:57.475472 3868 cni.go:403] [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default] I0327 10:54:57.551317 3868 cni.go:424] [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 4cbaa3a9942176c7370e782bb1abae161db82f40aeb8e0e431a36da7273829eb network default NAD default], result "{}", err I0327 10:54:57.606351 3868 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:54:57.606381 3868 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 10:54:57.607638 3868 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.40 I0327 10:54:57.607666 3868 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:54:58.356507 3868 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:54:58.356531 3868 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:54:58.364355 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 10:54:58.364380 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 10:54:58.392805 3868 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:54:58.392901 3868 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 10:54:58.411752 3868 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 18.876889ms I0327 10:54:58.411774 3868 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 18.904678ms I0327 10:54:58.413267 3868 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 20.482139ms, libovsdb time 1.052968ms I0327 10:54:58.752547 3868 cni.go:403] [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default] I0327 10:54:58.761429 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 2a90208fd8c3c6b, network: default, NAD default, SandboxID: "2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e", PCI device ID: , UID: "ddc62ec4-c793-456d-bab9-42f8fb5a5059", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0327 10:54:58.808178 3868 cni.go:424] [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:3c:73:44:3e:64\",\"name\":\"2a90208fd8c3c6b\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c08af492-715c-4dfc-8c8b-9632146427b7\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:54:59.613234 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=91 W0327 10:55:00.379014 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 10:55:00.382286 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:00.382310 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:14.066274 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:14.066296 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:14.067258 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 10:55:18.797155 3868 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-w674w] creating logical port tsf-keycloak_tsf-iam-w674w for pod on switch ip-10-0-2-4.ec2.internal I0327 10:55:18.797264 3868 kube.go:256] Updating pod tsf-keycloak/tsf-iam-w674w I0327 10:55:18.813641 3868 pod.go:62] [tsf-keycloak/tsf-iam-w674w] pod update took 16.41522ms I0327 10:55:18.813660 3868 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-w674w] addLogicalPort annotation time took 16.439628ms I0327 10:55:18.814928 3868 pods.go:271] [tsf-keycloak/tsf-iam-w674w] addLogicalPort took 17.790975ms, libovsdb time 839.05µs I0327 10:55:19.157251 3868 cni.go:403] [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default] I0327 10:55:19.166458 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-w674w, hostIfaceName: d461fc5582ee687, network: default, NAD default, SandboxID: "d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87", PCI device ID: , UID: "394c9e9f-42f1-4926-a903-fb48543063b6", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0327 10:55:19.213449 3868 cni.go:424] [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:9d:9a:ea:9b:95\",\"name\":\"d461fc5582ee687\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8b2b1785-3e72-4f81-943b-4a820685d6c7\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:55:23.710752 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0327 10:55:29.719518 3868 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:55:29.720219 3868 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:55:36.603122 3868 cni.go:403] [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default] I0327 10:55:36.690327 3868 cni.go:424] [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-w674w d461fc5582ee6878600bb7b5e5f7d7e7fa10ca1ed7387b21d7a924822328de87 network default NAD default], result "{}", err I0327 10:55:36.746487 3868 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-w674w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:55:36.746523 3868 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-w674w I0327 10:55:36.747911 3868 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-w674w, ips: 10.128.0.42 I0327 10:55:36.747943 3868 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-w674w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:55:37.780636 3868 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-w674w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:55:37.780660 3868 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-w674w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:55:38.087486 3868 cni.go:403] [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default] I0327 10:55:38.173976 3868 cni.go:424] [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 2a90208fd8c3c6b217a2f65927a7144fdfa767da86bfe48e3e9f4433be65b04e network default NAD default], result "{}", err I0327 10:55:38.237992 3868 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:55:38.238021 3868 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 10:55:38.239355 3868 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.41 I0327 10:55:38.239386 3868 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:55:38.609820 3868 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:55:38.609847 3868 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:55:38.619858 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 10:55:38.619881 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 10:55:38.651494 3868 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:55:38.651584 3868 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 10:55:38.665817 3868 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 14.257933ms I0327 10:55:38.665843 3868 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 14.287017ms I0327 10:55:38.667080 3868 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 15.605293ms, libovsdb time 831.157µs I0327 10:55:39.014924 3868 cni.go:403] [tsf-keycloak/keycloak-0 c00a9a08660b4b1c78940a5f83ae69bd7c7b193a4899570debc0c319598e0d79 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 c00a9a08660b4b1c78940a5f83ae69bd7c7b193a4899570debc0c319598e0d79 network default NAD default] I0327 10:55:39.023433 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: c00a9a08660b4b1, network: default, NAD default, SandboxID: "c00a9a08660b4b1c78940a5f83ae69bd7c7b193a4899570debc0c319598e0d79", PCI device ID: , UID: "3f2837b0-aa42-4871-979d-6738996e1ef3", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0327 10:55:39.070502 3868 cni.go:424] [tsf-keycloak/keycloak-0 c00a9a08660b4b1c78940a5f83ae69bd7c7b193a4899570debc0c319598e0d79 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 c00a9a08660b4b1c78940a5f83ae69bd7c7b193a4899570debc0c319598e0d79 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:92:39:70:0a:bb\",\"name\":\"c00a9a08660b4b1\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8caede1a-a6e8-4909-b409-9f5d3751b2e5\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:55:40.642620 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 10:55:40.642620 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:40.642828 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:53.369630 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:53.369658 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 10:55:53.375574 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 10:56:29.808565 3868 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:56:29.808593 3868 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:56:29.816633 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 10:56:29.816654 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 10:56:39.087361 3868 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:56:39.087933 3868 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:56:39.115353 3868 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:56:39.115374 3868 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:56:39.123374 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 10:56:39.123398 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 10:56:42.431610 3868 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-7788bd658f-lw8s2] creating logical port tsf-tas_trillian-logsigner-7788bd658f-lw8s2 for pod on switch ip-10-0-2-4.ec2.internal I0327 10:56:42.431782 3868 kube.go:256] Updating pod tsf-tas/trillian-logsigner-7788bd658f-lw8s2 I0327 10:56:42.447931 3868 pod.go:62] [tsf-tas/trillian-logsigner-7788bd658f-lw8s2] pod update took 16.172074ms I0327 10:56:42.448024 3868 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logsigner-7788bd658f-lw8s2] addLogicalPort annotation time took 16.309369ms I0327 10:56:42.449511 3868 pods.go:271] [tsf-tas/trillian-logsigner-7788bd658f-lw8s2] addLogicalPort took 17.916537ms, libovsdb time 896.271µs I0327 10:56:43.396595 3868 cni.go:403] [tsf-tas/trillian-logsigner-7788bd658f-lw8s2 1b9506c79f8caee863eb9874a8e21e81893894f4b8c2c6867f0f47c8efc51470 network default NAD default] ADD starting CNI request [tsf-tas/trillian-logsigner-7788bd658f-lw8s2 1b9506c79f8caee863eb9874a8e21e81893894f4b8c2c6867f0f47c8efc51470 network default NAD default] I0327 10:56:43.405182 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-7788bd658f-lw8s2, hostIfaceName: 1b9506c79f8caee, network: default, NAD default, SandboxID: "1b9506c79f8caee863eb9874a8e21e81893894f4b8c2c6867f0f47c8efc51470", PCI device ID: , UID: "b5cb07fd-625e-4e28-8a0f-b4c564518b7a", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0327 10:56:43.453448 3868 cni.go:424] [tsf-tas/trillian-logsigner-7788bd658f-lw8s2 1b9506c79f8caee863eb9874a8e21e81893894f4b8c2c6867f0f47c8efc51470 network default NAD default] ADD finished CNI request [tsf-tas/trillian-logsigner-7788bd658f-lw8s2 1b9506c79f8caee863eb9874a8e21e81893894f4b8c2c6867f0f47c8efc51470 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:c9:96:13:fc:67\",\"name\":\"1b9506c79f8caee\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/439fb8bd-be30-4833-81e0-d1f75b1a4c65\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:56:45.490796 3868 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 10:56:45.490819 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0327 10:56:53.004619 3868 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 10:56:54.076853 3868 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0327 10:56:57.069610 3868 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 10:56:57.191955 3868 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-mlnvm-pgkrt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:56:57.192475 3868 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-mlnvm-pgkrt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:56:58.088819 3868 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-mlnvm-pgkrt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:56:58.088842 3868 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-mlnvm-pgkrt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:08.736947 3868 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 10:57:20.615738 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=51 W0327 10:57:28.142192 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0327 10:57:29.384126 3868 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:29.384660 3868 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:29.405634 3868 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:29.405653 3868 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:29.413227 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 10:57:29.413249 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 10:57:30.406263 3868 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-gvgt7-j425z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:30.406841 3868 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-gvgt7-j425z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:31.290290 3868 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-gvgt7-j425z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:31.290311 3868 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-gvgt7-j425z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 10:57:38.329785 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0327 10:57:38.474220 3868 obj_retry.go:464] Detected object tsf/patch-tekton-config-457tt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:38.474876 3868 obj_retry.go:464] Detected object tsf/patch-tekton-config-457tt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:38.928791 3868 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-w6bwq-57mmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:38.929369 3868 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-w6bwq-57mmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:39.348425 3868 obj_retry.go:458] Detected object tsf/patch-tekton-config-457tt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:39.348446 3868 obj_retry.go:458] Detected object tsf/patch-tekton-config-457tt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:39.801651 3868 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-w6bwq-57mmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:39.801678 3868 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-w6bwq-57mmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:43.677619 3868 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-77648f579f-vhttc] creating logical port tsf-tas_rekor-redis-77648f579f-vhttc for pod on switch ip-10-0-2-4.ec2.internal I0327 10:57:43.677729 3868 kube.go:256] Updating pod tsf-tas/rekor-redis-77648f579f-vhttc I0327 10:57:43.691331 3868 pod.go:62] [tsf-tas/rekor-redis-77648f579f-vhttc] pod update took 13.638983ms I0327 10:57:43.691359 3868 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-77648f579f-vhttc] addLogicalPort annotation time took 13.668214ms I0327 10:57:43.692780 3868 pods.go:271] [tsf-tas/rekor-redis-77648f579f-vhttc] addLogicalPort took 15.183342ms, libovsdb time 943.677µs I0327 10:57:44.658681 3868 cni.go:403] [tsf-tas/rekor-redis-77648f579f-vhttc 78016b6b21e3a1d148296690a44be86b513abcdc58016f259261ff2025b2a572 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-77648f579f-vhttc 78016b6b21e3a1d148296690a44be86b513abcdc58016f259261ff2025b2a572 network default NAD default] I0327 10:57:44.667423 3868 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-77648f579f-vhttc, hostIfaceName: 78016b6b21e3a1d, network: default, NAD default, SandboxID: "78016b6b21e3a1d148296690a44be86b513abcdc58016f259261ff2025b2a572", PCI device ID: , UID: "a2ac7ad4-7db9-4146-a260-db65556571dc", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0327 10:57:44.715991 3868 cni.go:424] [tsf-tas/rekor-redis-77648f579f-vhttc 78016b6b21e3a1d148296690a44be86b513abcdc58016f259261ff2025b2a572 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-77648f579f-vhttc 78016b6b21e3a1d148296690a44be86b513abcdc58016f259261ff2025b2a572 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:dc:db:e0:b3:93\",\"name\":\"78016b6b21e3a1d\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92736321-3b79-408b-954f-8fafbd2a5fb4\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:57:47.278006 3868 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0327 10:57:48.393215 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0327 10:57:49.531433 3868 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-kf2vd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:49.531973 3868 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-kf2vd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:50.436521 3868 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-kf2vd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:50.436542 3868 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-kf2vd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:50.478348 3868 obj_retry.go:458] Detected object tsf/patch-tekton-config-457tt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:50.478369 3868 obj_retry.go:458] Detected object tsf/patch-tekton-config-457tt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:50.483825 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-457tt I0327 10:57:50.483852 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-457tt W0327 10:57:51.859945 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0327 10:57:59.325483 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0327 10:57:59.600839 3868 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:59.601373 3868 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:57:59.619475 3868 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:59.619497 3868 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:57:59.623123 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 10:57:59.623139 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 10:58:03.715937 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 10:58:04.636326 3868 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:04.636883 3868 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:04.662339 3868 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:04.662362 3868 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:04.666680 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 10:58:04.666705 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 10:58:11.684481 3868 obj_retry.go:464] Detected object tsf-tpa/create-db-68tx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:11.685069 3868 obj_retry.go:464] Detected object tsf-tpa/create-db-68tx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:12.708864 3868 obj_retry.go:458] Detected object tsf-tpa/create-db-68tx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:12.708890 3868 obj_retry.go:458] Detected object tsf-tpa/create-db-68tx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:15.734159 3868 obj_retry.go:464] Detected object tsf-tpa/migrate-db-bjtt9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:15.734731 3868 obj_retry.go:464] Detected object tsf-tpa/migrate-db-bjtt9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:15.856406 3868 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bjtt9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:15.856429 3868 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bjtt9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:21.169648 3868 obj_retry.go:464] Detected object tsf-tpa/create-importers-8hwlt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:21.170210 3868 obj_retry.go:464] Detected object tsf-tpa/create-importers-8hwlt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:22.193972 3868 obj_retry.go:458] Detected object tsf-tpa/create-importers-8hwlt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:22.193994 3868 obj_retry.go:458] Detected object tsf-tpa/create-importers-8hwlt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 10:58:23.661425 3868 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0327 10:58:24.191487 3868 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-25944-dvjkx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:24.192127 3868 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-25944-dvjkx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:25.066355 3868 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-25944-dvjkx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:25.066378 3868 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-25944-dvjkx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:25.709668 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 W0327 10:58:30.088615 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0327 10:58:51.933437 3868 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:51.933989 3868 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:51.990025 3868 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:51.990044 3868 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:51.998918 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 10:58:51.998941 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 10:58:52.866884 3868 namespace.go:98] [image-controller] adding namespace I0327 10:58:52.867791 3868 namespace.go:102] [image-controller] adding namespace took 884.014µs I0327 10:58:52.870251 3868 namespace.go:98] [integration-service] adding namespace I0327 10:58:52.870269 3868 namespace.go:98] [build-service] adding namespace I0327 10:58:52.870943 3868 namespace.go:102] [integration-service] adding namespace took 677.616µs I0327 10:58:52.871459 3868 namespace.go:102] [build-service] adding namespace took 1.177303ms I0327 10:58:52.933826 3868 namespace.go:142] [image-controller] updating namespace I0327 10:58:52.955844 3868 namespace.go:142] [image-controller] updating namespace I0327 10:58:52.975494 3868 namespace.go:142] [integration-service] updating namespace I0327 10:58:53.000210 3868 namespace.go:142] [integration-service] updating namespace I0327 10:58:53.008451 3868 namespace.go:142] [build-service] updating namespace I0327 10:58:53.025230 3868 namespace.go:142] [build-service] updating namespace I0327 10:58:53.081227 3868 namespace.go:142] [build-service] updating namespace I0327 10:58:53.098361 3868 namespace.go:142] [integration-service] updating namespace I0327 10:58:53.136094 3868 namespace.go:98] [release-service] adding namespace I0327 10:58:53.136953 3868 namespace.go:102] [release-service] adding namespace took 843.518µs I0327 10:58:53.169367 3868 namespace.go:142] [konflux-ui] updating namespace I0327 10:58:53.184426 3868 namespace.go:142] [release-service] updating namespace I0327 10:58:53.213212 3868 namespace.go:142] [release-service] updating namespace I0327 10:58:53.282097 3868 namespace.go:98] [konflux-info] adding namespace I0327 10:58:53.282986 3868 namespace.go:102] [konflux-info] adding namespace took 870.376µs I0327 10:58:53.322578 3868 namespace.go:98] [namespace-lister] adding namespace I0327 10:58:53.323436 3868 namespace.go:102] [namespace-lister] adding namespace took 839.139µs I0327 10:58:53.354239 3868 namespace.go:142] [konflux-info] updating namespace I0327 10:58:53.362220 3868 namespace.go:142] [image-controller] updating namespace I0327 10:58:53.362259 3868 namespace.go:98] [enterprise-contract-service] adding namespace I0327 10:58:53.362962 3868 namespace.go:102] [enterprise-contract-service] adding namespace took 689.111µs I0327 10:58:53.409048 3868 namespace.go:142] [konflux-info] updating namespace I0327 10:58:53.458216 3868 namespace.go:142] [namespace-lister] updating namespace I0327 10:58:53.507705 3868 namespace.go:142] [namespace-lister] updating namespace I0327 10:58:53.565072 3868 namespace.go:142] [enterprise-contract-service] updating namespace I0327 10:58:53.657172 3868 namespace.go:98] [default-tenant] adding namespace I0327 10:58:53.658125 3868 namespace.go:102] [default-tenant] adding namespace took 931.518µs I0327 10:58:53.720451 3868 namespace.go:142] [enterprise-contract-service] updating namespace I0327 10:58:53.984027 3868 namespace.go:142] [default-tenant] updating namespace I0327 10:58:54.027031 3868 namespace.go:98] [segment-bridge] adding namespace I0327 10:58:54.027882 3868 namespace.go:102] [segment-bridge] adding namespace took 832.775µs I0327 10:58:54.086210 3868 namespace.go:142] [default-tenant] updating namespace I0327 10:58:54.281156 3868 namespace.go:142] [segment-bridge] updating namespace I0327 10:58:54.371231 3868 namespace.go:142] [segment-bridge] updating namespace I0327 10:58:54.565653 3868 namespace.go:142] [build-service] updating namespace I0327 10:58:54.576674 3868 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 10:58:54.577251 3868 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 10:58:54.577271 3868 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 10:58:54.579785 3868 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 10:58:54.619100 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0327 10:58:54.688326 3868 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 10:58:54.688356 3868 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 10:58:54.689207 3868 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 10:58:54.742455 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0327 10:58:54.971160 3868 namespace.go:142] [enterprise-contract-service] updating namespace I0327 10:58:55.259702 3868 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-p6dkq for pod on switch ip-10-0-2-4.ec2.internal I0327 10:58:55.259867 3868 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq I0327 10:58:55.304851 3868 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq] pod update took 45.006946ms I0327 10:58:55.304875 3868 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq] addLogicalPort annotation time took 45.037729ms I0327 10:58:55.306150 3868 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq] addLogicalPort took 46.470483ms, libovsdb time 891.456µs I0327 10:58:55.320188 3868 namespace.go:142] [image-controller] updating namespace I0327 10:58:55.585884 3868 namespace.go:142] [integration-service] updating namespace I0327 10:58:55.708245 3868 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-rxbzt] creating logical port integration-service_integration-service-controller-manager-759977d69f-rxbzt for pod on switch ip-10-0-2-4.ec2.internal I0327 10:58:55.708357 3868 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-rxbzt I0327 10:58:55.731353 3868 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-rxbzt] pod update took 23.025131ms I0327 10:58:55.731374 3868 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-rxbzt] addLogicalPort annotation time took 23.055212ms I0327 10:58:55.732805 3868 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-rxbzt] addLogicalPort took 24.573481ms, libovsdb time 911.438µs I0327 10:58:55.814323 3868 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-68fkh] creating logical port build-service_build-service-controller-manager-7df75dd4df-68fkh for pod on switch ip-10-0-2-4.ec2.internal I0327 10:58:55.814407 3868 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-68fkh I0327 10:58:55.815038 3868 namespace.go:142] [konflux-info] updating namespace I0327 10:58:55.865045 3868 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-68fkh I0327 10:58:55.904089 3868 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-68fkh] pod update took 89.701274ms I0327 10:58:55.904118 3868 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-68fkh] addLogicalPort annotation time took 89.736955ms I0327 10:58:55.905381 3868 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-68fkh] addLogicalPort took 91.072557ms, libovsdb time 846.318µs I0327 10:58:55.917086 3868 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default] I0327 10:58:55.925974 3868 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-p6dkq, hostIfaceName: 85928137c34fe18, network: default, NAD default, SandboxID: "85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab", PCI device ID: , UID: "068bfa63-e95c-4c23-a4f5-68af92faa99b", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0327 10:58:55.974953 3868 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:8e:34:00:56:9b\",\"name\":\"85928137c34fe18\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/afc16761-eceb-442a-946a-9016350dcf88\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:58:56.015191 3868 namespace.go:142] [namespace-lister] updating namespace I0327 10:58:56.163358 3868 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-68fkh 5df6a85a0ae424673b5624b35e4526228c0b4e42b3eaab387765f1dacbfe4d47 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-68fkh 5df6a85a0ae424673b5624b35e4526228c0b4e42b3eaab387765f1dacbfe4d47 network default NAD default] I0327 10:58:56.172115 3868 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-68fkh, hostIfaceName: 5df6a85a0ae4246, network: default, NAD default, SandboxID: "5df6a85a0ae424673b5624b35e4526228c0b4e42b3eaab387765f1dacbfe4d47", PCI device ID: , UID: "956e17ab-6b4f-40ad-9e28-130e459d4b3c", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0327 10:58:56.212172 3868 namespace.go:142] [release-service] updating namespace I0327 10:58:56.220125 3868 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-68fkh 5df6a85a0ae424673b5624b35e4526228c0b4e42b3eaab387765f1dacbfe4d47 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-68fkh 5df6a85a0ae424673b5624b35e4526228c0b4e42b3eaab387765f1dacbfe4d47 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:b2:08:05:c7:aa\",\"name\":\"5df6a85a0ae4246\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ac957e11-a171-4e44-8ebb-2f2c9d4ec2fc\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:58:56.317510 3868 namespace.go:142] [build-service] updating namespace I0327 10:58:56.351391 3868 namespace.go:142] [enterprise-contract-service] updating namespace I0327 10:58:56.353383 3868 namespace.go:142] [image-controller] updating namespace I0327 10:58:56.353419 3868 namespace.go:142] [integration-service] updating namespace I0327 10:58:56.395100 3868 namespace.go:142] [konflux-info] updating namespace I0327 10:58:56.413115 3868 namespace.go:142] [namespace-lister] updating namespace I0327 10:58:56.418161 3868 namespace.go:142] [release-service] updating namespace I0327 10:58:56.487818 3868 namespace.go:142] [build-service] updating namespace I0327 10:58:56.550176 3868 namespace.go:142] [enterprise-contract-service] updating namespace I0327 10:58:56.601406 3868 namespace.go:142] [image-controller] updating namespace I0327 10:58:56.694169 3868 namespace.go:142] [integration-service] updating namespace I0327 10:58:56.787425 3868 namespace.go:142] [konflux-info] updating namespace I0327 10:58:56.898103 3868 namespace.go:142] [namespace-lister] updating namespace I0327 10:58:56.980630 3868 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-45h7j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:56.981284 3868 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-45h7j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:58:56.987171 3868 namespace.go:142] [release-service] updating namespace I0327 10:58:57.012624 3868 base_network_controller_pods.go:487] [default/konflux-ui/proxy-685887f4d9-wdm2t] creating logical port konflux-ui_proxy-685887f4d9-wdm2t for pod on switch ip-10-0-2-4.ec2.internal I0327 10:58:57.012821 3868 kube.go:256] Updating pod konflux-ui/proxy-685887f4d9-wdm2t I0327 10:58:57.093344 3868 kube.go:256] Updating pod konflux-ui/proxy-685887f4d9-wdm2t I0327 10:58:57.170196 3868 pod.go:62] [konflux-ui/proxy-685887f4d9-wdm2t] pod update took 157.480832ms I0327 10:58:57.170226 3868 base_network_controller_pods.go:937] [default/konflux-ui/proxy-685887f4d9-wdm2t] addLogicalPort annotation time took 157.516053ms I0327 10:58:57.171878 3868 pods.go:271] [konflux-ui/proxy-685887f4d9-wdm2t] addLogicalPort took 159.245241ms, libovsdb time 1.068817ms I0327 10:58:58.004829 3868 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-45h7j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:58.004857 3868 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-45h7j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:58:58.180380 3868 namespace.go:142] [default-tenant] updating namespace I0327 10:58:58.184090 3868 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-4.ec2.internal I0327 10:58:58.184187 3868 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0327 10:58:58.201990 3868 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 17.830099ms I0327 10:58:58.202009 3868 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 17.853807ms I0327 10:58:58.203363 3868 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 19.283554ms, libovsdb time 841.65µs W0327 10:58:58.266819 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0327 10:58:58.376168 3868 namespace.go:142] [segment-bridge] updating namespace I0327 10:58:58.499704 3868 namespace.go:142] [default-tenant] updating namespace I0327 10:58:58.514927 3868 namespace.go:142] [segment-bridge] updating namespace I0327 10:58:58.600404 3868 namespace.go:142] [default-tenant] updating namespace I0327 10:58:58.699521 3868 namespace.go:142] [segment-bridge] updating namespace I0327 10:58:59.588056 3868 cni.go:403] [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default] I0327 10:58:59.596791 3868 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: c0659c48b5ed489, network: default, NAD default, SandboxID: "c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875", PCI device ID: , UID: "8100df0a-7529-4a9a-b247-4531393ebfda", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0327 10:58:59.647847 3868 cni.go:424] [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:62:e0:3a:37:9a\",\"name\":\"c0659c48b5ed489\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78be94d4-9b7c-452c-b2cd-8495cc7e7b1f\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:59:00.661931 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 10:59:00.691635 3868 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0327 10:59:00.972350 3868 cni.go:403] [konflux-ui/proxy-685887f4d9-wdm2t a5d6208144f78d372766db2680d49cf52e743aafeebc206dd81af20ed4e9e14e network default NAD default] ADD starting CNI request [konflux-ui/proxy-685887f4d9-wdm2t a5d6208144f78d372766db2680d49cf52e743aafeebc206dd81af20ed4e9e14e network default NAD default] I0327 10:59:00.982418 3868 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-685887f4d9-wdm2t, hostIfaceName: a5d6208144f78d3, network: default, NAD default, SandboxID: "a5d6208144f78d372766db2680d49cf52e743aafeebc206dd81af20ed4e9e14e", PCI device ID: , UID: "97b31065-f564-45d1-b91c-927a180c6e1c", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0327 10:59:01.036379 3868 cni.go:424] [konflux-ui/proxy-685887f4d9-wdm2t a5d6208144f78d372766db2680d49cf52e743aafeebc206dd81af20ed4e9e14e network default NAD default] ADD finished CNI request [konflux-ui/proxy-685887f4d9-wdm2t a5d6208144f78d372766db2680d49cf52e743aafeebc206dd81af20ed4e9e14e network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:43:4a:93:78:a2\",\"name\":\"a5d6208144f78d3\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2efd259a-4c35-47e4-a9ae-6c0cd3c87d34\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 10:59:03.565493 3868 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default] I0327 10:59:03.585858 3868 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-rxbzt, hostIfaceName: 4eaa9c1e120e1fa, network: default, NAD default, SandboxID: "4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14", PCI device ID: , UID: "f37466ac-a47a-4977-a295-de8af29afef5", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0327 10:59:03.637528 3868 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:89:19:03:10:01\",\"name\":\"4eaa9c1e120e1fa\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/061e4c8a-5fa7-4864-b2ba-76b3bb6a1d33\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 10:59:05.299807 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 10:59:05.300967 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 10:59:06.312551 3868 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 10:59:06.711023 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:06.711113 3868 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:10.373090 3868 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:10.373110 3868 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:10.398263 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 10:59:10.401695 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 10:59:10.764366 3868 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 10:59:10.764957 3868 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 10:59:11.668956 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 10:59:11.697927 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 10:59:16.313590 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 10:59:16.313688 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 10:59:16.313774 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 10:59:16.313789 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0327 10:59:16.499213 3868 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default] I0327 10:59:16.576031 3868 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-rxbzt 4eaa9c1e120e1fa13ab4957a7df2c886a5302a50e14344614fba13aa4fe96b14 network default NAD default], result "{}", err I0327 10:59:16.620256 3868 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-rxbzt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 10:59:16.620296 3868 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-rxbzt I0327 10:59:16.621777 3868 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-rxbzt, ips: 10.128.0.47 I0327 10:59:16.621809 3868 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-rxbzt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 10:59:16.765766 3868 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-rxbzt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:59:16.765793 3868 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-rxbzt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 10:59:16.773744 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-rxbzt I0327 10:59:16.773769 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-rxbzt I0327 10:59:30.626328 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=112 I0327 10:59:31.722002 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0327 10:59:40.750833 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 W0327 10:59:47.406971 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:47.406993 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:47.407006 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:47.407012 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 10:59:48.706217 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 W0327 10:59:49.805418 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 10:59:49.805441 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 11:00:00.185545 3868 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576820-x8k56 for pod on switch ip-10-0-2-4.ec2.internal I0327 11:00:00.185637 3868 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 I0327 11:00:00.198027 3868 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56] pod update took 12.409689ms I0327 11:00:00.198055 3868 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56] addLogicalPort annotation time took 12.443189ms I0327 11:00:00.199576 3868 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56] addLogicalPort took 14.045657ms, libovsdb time 1.022193ms I0327 11:00:00.553442 3868 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default] I0327 11:00:00.562256 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576820-x8k56, hostIfaceName: 722287f7a0f0d25, network: default, NAD default, SandboxID: "722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3", PCI device ID: , UID: "92dbae05-dd82-4ff5-85ab-134d3c56a511", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0327 11:00:00.611751 3868 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:de:05:38:f4:55\",\"name\":\"722287f7a0f0d25\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5648c2b9-6967-4bb0-a15d-b55a4b2f9770\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:00:04.034942 3868 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default] I0327 11:00:04.123538 3868 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 722287f7a0f0d25f169589a3eb878f4b48cbf900fe08916dcf37e8b103a3a5f3 network default NAD default], result "{}", err I0327 11:00:04.174275 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:00:04.174315 3868 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 I0327 11:00:04.175932 3868 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56, ips: 10.128.0.51 I0327 11:00:04.175962 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:00:05.053172 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:00:05.053196 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576820-x8k56 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:00:07.742605 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0327 11:00:19.343643 3868 obj_retry.go:464] Detected object konflux-ui/proxy-5cfc9c98bd-m5t9q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:00:19.344209 3868 obj_retry.go:464] Detected object konflux-ui/proxy-5cfc9c98bd-m5t9q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:00:19.766081 3868 obj_retry.go:458] Detected object konflux-ui/proxy-5cfc9c98bd-m5t9q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:00:19.766104 3868 obj_retry.go:458] Detected object konflux-ui/proxy-5cfc9c98bd-m5t9q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:00:19.775412 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5cfc9c98bd-m5t9q I0327 11:00:19.775432 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5cfc9c98bd-m5t9q I0327 11:00:30.618044 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=52 I0327 11:00:36.211675 3868 cni.go:403] [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default] I0327 11:00:36.285734 3868 cni.go:424] [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux c0659c48b5ed489b9c7abe42198cc0e2262259c0d4a9bf2f79317447883af875 network default NAD default], result "{}", err I0327 11:00:36.338817 3868 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:00:36.338852 3868 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0327 11:00:36.340272 3868 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.50 I0327 11:00:36.340300 3868 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:00:36.357654 3868 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:00:36.357673 3868 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:00:36.365560 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 11:00:36.365589 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 11:01:05.335833 3868 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-tfbbx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:01:05.336430 3868 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-tfbbx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:01:06.045564 3868 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-tfbbx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:01:06.045597 3868 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-tfbbx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:01:06.055551 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-tfbbx I0327 11:01:06.055573 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-tfbbx I0327 11:01:08.732508 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0327 11:01:20.264769 3868 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default] I0327 11:01:20.346200 3868 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq 85928137c34fe180a4500682784bf7b9732115a95df3f7b8224f990cd5454aab network default NAD default], result "{}", err I0327 11:01:20.399424 3868 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:01:20.399535 3868 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq I0327 11:01:20.400808 3868 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq, ips: 10.128.0.46 I0327 11:01:20.400834 3868 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:01:20.741182 3868 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:01:20.741209 3868 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:01:20.745113 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq I0327 11:01:20.745147 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-p6dkq I0327 11:03:41.626742 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=439 I0327 11:03:51.739670 3868 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0327 11:03:59.616796 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=96 I0327 11:05:21.730003 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0327 11:05:23.712777 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 11:05:53.725215 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0327 11:06:00.753776 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 11:06:03.719392 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0327 11:06:12.621361 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0327 11:07:00.618459 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=63 I0327 11:07:28.745237 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 11:07:31.627225 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0327 11:07:45.709135 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0327 11:07:56.629790 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0327 11:08:10.745629 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0327 11:08:12.058494 3868 obj_retry.go:458] Detected object tsf-tpa/create-db-68tx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:08:12.058547 3868 obj_retry.go:458] Detected object tsf-tpa/create-db-68tx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:08:12.064229 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-68tx9 I0327 11:08:12.064255 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-68tx9 I0327 11:08:15.037319 3868 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bjtt9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:08:15.037364 3868 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bjtt9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:08:15.043581 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-bjtt9 I0327 11:08:15.043602 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-bjtt9 I0327 11:08:22.037800 3868 obj_retry.go:458] Detected object tsf-tpa/create-importers-8hwlt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:08:22.037857 3868 obj_retry.go:458] Detected object tsf-tpa/create-importers-8hwlt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:08:22.043504 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-8hwlt I0327 11:08:22.043534 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-8hwlt I0327 11:10:00.202336 3868 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576830-6gxkj for pod on switch ip-10-0-2-4.ec2.internal I0327 11:10:00.202447 3868 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj I0327 11:10:00.232403 3868 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj] pod update took 29.977098ms I0327 11:10:00.232429 3868 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj] addLogicalPort annotation time took 30.004935ms I0327 11:10:00.233749 3868 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj] addLogicalPort took 31.424433ms, libovsdb time 862.978µs I0327 11:10:00.565279 3868 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default] I0327 11:10:00.574438 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576830-6gxkj, hostIfaceName: bbcf7c9c3cd29a0, network: default, NAD default, SandboxID: "bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1", PCI device ID: , UID: "412dd90e-8652-485f-8f41-9d1bb1ca9b43", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0327 11:10:00.619869 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0327 11:10:00.830350 3868 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:ee:70:f0:df:41\",\"name\":\"bbcf7c9c3cd29a0\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cde0ca4-37f5-4e38-a72f-908d107f6b3a\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:10:04.245444 3868 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default] I0327 11:10:04.332950 3868 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj bbcf7c9c3cd29a0f42d15d306522dfb212eed0fad4de6706a646738bc1b473c1 network default NAD default], result "{}", err I0327 11:10:04.386091 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:10:04.386144 3868 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj I0327 11:10:04.388158 3868 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj, ips: 10.128.0.52 I0327 11:10:04.388203 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:10:05.263296 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:10:05.263357 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576830-6gxkj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:10:22.629676 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=24 I0327 11:10:58.735614 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0327 11:11:25.623889 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0327 11:11:30.742689 3868 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 11:11:48.740210 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0327 11:13:15.756237 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 11:13:56.721842 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0327 11:14:02.711766 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0327 11:14:08.715080 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0327 11:14:47.859105 3868 namespace.go:98] [default-tenant-managed] adding namespace I0327 11:14:47.859975 3868 namespace.go:102] [default-tenant-managed] adding namespace took 846.596µs I0327 11:14:47.886484 3868 namespace.go:142] [default-tenant-managed] updating namespace I0327 11:14:47.901836 3868 namespace.go:142] [default-tenant-managed] updating namespace I0327 11:14:48.876185 3868 namespace.go:142] [default-tenant-managed] updating namespace I0327 11:14:49.016806 3868 namespace.go:142] [default-tenant-managed] updating namespace I0327 11:14:49.098366 3868 namespace.go:142] [default-tenant-managed] updating namespace I0327 11:15:30.747458 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0327 11:15:36.777311 3868 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-pjnwj for pod on switch ip-10-0-2-4.ec2.internal I0327 11:15:36.777426 3868 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj I0327 11:15:36.791010 3868 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj] pod update took 13.614592ms I0327 11:15:36.791030 3868 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj] addLogicalPort annotation time took 13.639468ms I0327 11:15:36.792267 3868 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj] addLogicalPort took 14.969597ms, libovsdb time 856.233µs I0327 11:15:37.137971 3868 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default] I0327 11:15:37.147332 3868 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-pjnwj, hostIfaceName: d805f440664d09c, network: default, NAD default, SandboxID: "d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750", PCI device ID: , UID: "7a674e12-b988-426e-bfbb-d1ab84e4d5ae", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0327 11:15:37.202168 3868 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:d8:38:98:33:b6\",\"name\":\"d805f440664d09c\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bcf03bb-499b-4769-ac1c-3459f0db3e35\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:15:39.727642 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0327 11:15:41.621212 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=38 I0327 11:15:49.366496 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:15:49.367090 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 11:15:51.177003 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177026 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177032 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177037 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177050 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177055 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177059 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:51.177063 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:52.045783 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:52.045807 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:52.045813 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 11:15:52.045818 3868 util.go:959] Failed to build node endpoints for node ip-10-0-2-4.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0327 11:16:07.632261 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0327 11:16:16.279556 3868 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod] creating logical port default-tenant_tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod for pod on switch ip-10-0-2-4.ec2.internal I0327 11:16:16.279748 3868 kube.go:256] Updating pod default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod I0327 11:16:16.310152 3868 pod.go:62] [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod] pod update took 30.464389ms I0327 11:16:16.310177 3868 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod] addLogicalPort annotation time took 30.49708ms I0327 11:16:16.311480 3868 pods.go:271] [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod] addLogicalPort took 31.94378ms, libovsdb time 885.017µs I0327 11:16:16.504381 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:16:16.504934 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:16:16.646211 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default] I0327 11:16:16.654769 3868 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod, hostIfaceName: 875d1d7283f1cf2, network: default, NAD default, SandboxID: "875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618", PCI device ID: , UID: "8d8afa8b-df86-4e48-9fc0-64f7401d08db", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0327 11:16:16.703845 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:64:c5:73:85:63\",\"name\":\"875d1d7283f1cf2\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/57256c0f-25f8-458d-94dd-239f57e694a4\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:16:40.398234 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default] I0327 11:16:40.488335 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod 875d1d7283f1cf2df164541e03d690629882bb8222649dbe1b856e9951bce618 network default NAD default], result "{}", err I0327 11:16:40.659627 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:16:40.659657 3868 pods.go:176] Deleting pod: default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod I0327 11:16:40.660910 3868 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod, ips: 10.128.0.54 I0327 11:16:40.660935 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:16:40.712960 3868 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod] creating logical port default-tenant_tsf-comp-wvul-on-push-5scxs-build-container-pod for pod on switch ip-10-0-2-4.ec2.internal I0327 11:16:40.713088 3868 kube.go:256] Updating pod default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod I0327 11:16:40.761061 3868 pod.go:62] [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod] pod update took 48.010256ms I0327 11:16:40.761083 3868 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod] addLogicalPort annotation time took 48.036876ms I0327 11:16:40.762575 3868 pods.go:271] [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod] addLogicalPort took 49.625894ms, libovsdb time 1.072855ms I0327 11:16:41.176382 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default] I0327 11:16:41.186851 3868 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-wvul-on-push-5scxs-build-container-pod, hostIfaceName: b94b7cd483a0491, network: default, NAD default, SandboxID: "b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c", PCI device ID: , UID: "60ba2941-12d2-47c5-841b-04e50fe293a9", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0327 11:16:41.235381 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:bd:7f:63:0a:bf\",\"name\":\"b94b7cd483a0491\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aa6aafc8-459a-4b9b-81e0-15dd220983a3\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:17:25.630320 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0327 11:18:03.749051 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 11:18:12.738701 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0327 11:18:43.742786 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0327 11:19:10.339478 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default] I0327 11:19:10.428912 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod b94b7cd483a04911a85e0b7dc8bfdbe785b86c9b789e7e9be3e47a77f3e7274c network default NAD default], result "{}", err I0327 11:19:10.551146 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:19:10.551213 3868 pods.go:176] Deleting pod: default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod I0327 11:19:10.552913 3868 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod, ips: 10.128.0.55 I0327 11:19:10.552940 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 11:19:19.020475 3868 duration.go:461] Config duration recorder: measurement expired for pod/integration-service/integration-service-controller-manager-759977d69f-rxbzt I0327 11:19:24.622833 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=14 I0327 11:19:34.714545 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0327 11:19:43.633087 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=86 I0327 11:19:50.502682 3868 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod] creating logical port default-tenant_tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod for pod on switch ip-10-0-2-4.ec2.internal I0327 11:19:50.502854 3868 kube.go:256] Updating pod default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod I0327 11:19:50.523450 3868 pod.go:62] [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod] pod update took 20.627116ms I0327 11:19:50.523472 3868 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod] addLogicalPort annotation time took 20.654101ms I0327 11:19:50.524870 3868 pods.go:271] [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod] addLogicalPort took 22.208439ms, libovsdb time 945.321µs I0327 11:19:50.588046 3868 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod] creating logical port default-tenant_tsf-comp-wvul-on-push-5scxs-clair-scan-pod for pod on switch ip-10-0-2-4.ec2.internal I0327 11:19:50.588166 3868 kube.go:256] Updating pod default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod I0327 11:19:50.614342 3868 pod.go:62] [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod] pod update took 26.208431ms I0327 11:19:50.614366 3868 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod] addLogicalPort annotation time took 26.236822ms I0327 11:19:50.615614 3868 pods.go:271] [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod] addLogicalPort took 27.588257ms, libovsdb time 835.146µs I0327 11:19:50.809192 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:19:50.809752 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:19:50.875628 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default] I0327 11:19:50.884478 3868 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod, hostIfaceName: 18792b24a8e03bf, network: default, NAD default, SandboxID: "18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1", PCI device ID: , UID: "faed8b9f-1fec-4cab-9cf6-19fa9a2c971e", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0327 11:19:50.935654 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:9e:4c:5f:0d:39\",\"name\":\"18792b24a8e03bf\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9eea8774-6d4d-4ea3-bff0-35553e1e4739\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:19:50.954789 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default] I0327 11:19:50.963320 3868 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-wvul-on-push-5scxs-clair-scan-pod, hostIfaceName: 7b37387636ad74e, network: default, NAD default, SandboxID: "7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af", PCI device ID: , UID: "68d59441-e742-4a22-8414-717fe8177116", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0327 11:19:51.014126 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:db:d5:51:04:0d\",\"name\":\"7b37387636ad74e\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffcf8537-eb9a-4609-a18b-c76aeaff4cc5\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:19:51.161402 3868 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod] creating logical port default-tenant_tsf-comp-wvul-on-push-5scxs-clamav-scan-pod for pod on switch ip-10-0-2-4.ec2.internal I0327 11:19:51.161559 3868 kube.go:256] Updating pod default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod I0327 11:19:51.185498 3868 pod.go:62] [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod] pod update took 23.967645ms I0327 11:19:51.185527 3868 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod] addLogicalPort annotation time took 24.002047ms I0327 11:19:51.186895 3868 pods.go:271] [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod] addLogicalPort took 25.507524ms, libovsdb time 905.113µs I0327 11:19:51.526126 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default] I0327 11:19:51.535058 3868 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-wvul-on-push-5scxs-clamav-scan-pod, hostIfaceName: 64b0696e3cc573d, network: default, NAD default, SandboxID: "64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298", PCI device ID: , UID: "5f173903-a354-4b37-8871-86a529b4f3c4", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0327 11:19:51.585851 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:3b:e6:de:0c:2d\",\"name\":\"64b0696e3cc573d\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/251670e4-2961-4175-81ae-a1c36d35d61d\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:20:03.718810 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576840-nbnzg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:03.719433 3868 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576840-nbnzg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:04.579442 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576840-nbnzg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:20:04.579468 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576840-nbnzg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:20:04.615479 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:20:04.615506 3868 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:20:04.618463 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 I0327 11:20:04.618497 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576810-mckv5 I0327 11:20:33.263258 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:33.263934 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:40.684477 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:40.685084 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:42.386056 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:42.386756 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:44.424785 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:44.425447 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:51.626750 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0327 11:20:51.750090 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0327 11:20:53.076085 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default] I0327 11:20:53.157830 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod 18792b24a8e03bfe3942d0981cdcbf3946277d7721286273ae5a8957117432d1 network default NAD default], result "{}", err I0327 11:20:53.267407 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:53.267472 3868 pods.go:176] Deleting pod: default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod I0327 11:20:53.268993 3868 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod, ips: 10.128.0.56 I0327 11:20:53.269021 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:57.113417 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default] I0327 11:20:57.208186 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod 64b0696e3cc573d1a279ab6d7e6dabaa8c502b72fff88cbfb14e33eb565d8298 network default NAD default], result "{}", err I0327 11:20:57.335012 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:57.335078 3868 pods.go:176] Deleting pod: default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod I0327 11:20:57.336466 3868 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod, ips: 10.128.0.58 I0327 11:20:57.336496 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:58.102333 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:20:58.102967 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:10.610051 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:10.610662 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:12.718299 3868 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 11:21:24.745155 3868 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0327 11:21:30.934733 3868 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default] DEL starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default] I0327 11:21:31.024260 3868 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default] DEL finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj d805f440664d09c3c02706a01aab045ff93d5beba49c1de7a7c446b9e6a06750 network default NAD default], result "{}", err I0327 11:21:31.218832 3868 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:31.218872 3868 pods.go:176] Deleting pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj I0327 11:21:31.220272 3868 pods.go:236] Attempting to release IPs for pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj, ips: 10.128.0.53 I0327 11:21:31.220306 3868 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:31.351147 3868 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:21:31.351170 3868 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:21:31.359005 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj I0327 11:21:31.359027 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-pjnwj I0327 11:21:33.014229 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:33.014806 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:36.879102 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:21:36.879688 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:22:41.635381 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0327 11:22:53.725019 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 11:23:09.759755 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0327 11:23:14.015665 3868 cni.go:403] [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default] I0327 11:23:14.111037 3868 cni.go:424] [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod 7b37387636ad74e0975541dde5993327b34fcaf60d0820de84b0df198d4d27af network default NAD default], result "{}", err I0327 11:23:14.194956 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:23:14.194991 3868 pods.go:176] Deleting pod: default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod I0327 11:23:14.196597 3868 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod, ips: 10.128.0.57 I0327 11:23:14.196629 3868 obj_retry.go:464] Detected object default-tenant/tsf-comp-wvul-on-push-5scxs-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:23:17.632990 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0327 11:23:18.382840 3868 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-rntf-z44kg-verify-pod] creating logical port default-tenant_my-integration-test-rntf-z44kg-verify-pod for pod on switch ip-10-0-2-4.ec2.internal I0327 11:23:18.383002 3868 kube.go:256] Updating pod default-tenant/my-integration-test-rntf-z44kg-verify-pod I0327 11:23:18.418118 3868 pod.go:62] [default-tenant/my-integration-test-rntf-z44kg-verify-pod] pod update took 35.157304ms I0327 11:23:18.418143 3868 base_network_controller_pods.go:937] [default/default-tenant/my-integration-test-rntf-z44kg-verify-pod] addLogicalPort annotation time took 35.187398ms I0327 11:23:18.419417 3868 pods.go:271] [default-tenant/my-integration-test-rntf-z44kg-verify-pod] addLogicalPort took 36.594688ms, libovsdb time 891.984µs I0327 11:23:18.751474 3868 cni.go:403] [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default] ADD starting CNI request [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default] I0327 11:23:18.760023 3868 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-rntf-z44kg-verify-pod, hostIfaceName: 0b535539c5fb783, network: default, NAD default, SandboxID: "0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc", PCI device ID: , UID: "85cc4626-10d5-46ab-99ea-02ac5eb1ea18", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0327 11:23:18.811865 3868 cni.go:424] [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default] ADD finished CNI request [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:0f:10:4f:5f:42\",\"name\":\"0b535539c5fb783\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/343b4332-a1fd-4e30-8970-d213050bfdb1\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 11:23:41.222268 3868 cni.go:403] [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default] DEL starting CNI request [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default] I0327 11:23:41.320691 3868 cni.go:424] [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default] DEL finished CNI request [default-tenant/my-integration-test-rntf-z44kg-verify-pod 0b535539c5fb783e0702e4f3c9f79b3b6ca763cdeecf8362c988db9d59396dcc network default NAD default], result "{}", err I0327 11:23:41.433824 3868 obj_retry.go:464] Detected object default-tenant/my-integration-test-rntf-z44kg-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:23:41.433866 3868 pods.go:176] Deleting pod: default-tenant/my-integration-test-rntf-z44kg-verify-pod I0327 11:23:41.435670 3868 pods.go:236] Attempting to release IPs for pod: default-tenant/my-integration-test-rntf-z44kg-verify-pod, ips: 10.128.0.59 I0327 11:23:41.435801 3868 obj_retry.go:464] Detected object default-tenant/my-integration-test-rntf-z44kg-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:12.133552 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:12.134212 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:36.243586 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:36.244210 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:46.331632 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:46.332201 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:47.349445 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:24:47.350001 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:02.483474 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:02.484057 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:08.549299 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:08.549847 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:14.730597 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 11:25:14.741593 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0327 11:25:15.745543 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0327 11:25:17.624279 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=72 I0327 11:25:25.635756 3868 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=266 I0327 11:25:32.787215 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:32.787765 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:58.006607 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:25:58.007181 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:26:22.711475 3868 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-d7a5e7d27c-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:26:22.712071 3868 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-d7a5e7d27c-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:26:23.126878 3868 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-d7a5e7d27c-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:26:23.126907 3868 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-d7a5e7d27c-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 11:26:23.138280 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-d7a5e7d27c-0 I0327 11:26:23.138304 3868 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-d7a5e7d27c-0 I0327 11:26:23.349755 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-verify-conforma-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 11:26:23.350327 3868 obj_retry.go:464] Detected object default-tenant-managed/managed-xq5wh-verify-conforma-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 11:26:34.752193 3868 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9