+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-187.ec2.internal ++ [[ -n ip-10-0-2-187.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-187.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260324-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260324-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260324-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0410 16:35:33.389937844 - disable conntrack on geneve port' I0410 16:35:33.389937844 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0410 16:35:33.407711900 - starting ovnkube-node + echo 'I0410 16:35:33.407711900 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-187.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-187.ec2.internal --init-node ip-10-0-2-187.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-187.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0410 16:35:33.452380 3933 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0410 16:35:33.452434 3933 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-1dfb53e96f.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0410 16:35:33.454236 3933 kube.go:423] Waiting for certificate I0410 16:35:33.454273 3933 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0410 16:35:33.454310 3933 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0410 16:35:33.455098 3933 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0410 16:35:33.482728 3933 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0410 16:35:33.482744 3933 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0410 16:35:33.485202 3933 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0410 16:35:33.488001 3933 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-85glc" I0410 16:35:33.495338 3933 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-85glc" I0410 16:35:33.495411 3933 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0410 16:35:34.455284 3933 kube.go:430] Certificate found I0410 16:35:34.455908 3933 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0410 16:35:34.457039 3933 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0410 16:35:34.498303 3933 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-11 16:30:33 +0000 UTC" deadline="2026-04-11 10:56:00.485223949 +0000 UTC" I0410 16:35:34.498340 3933 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h20m25.986897832s" I0410 16:35:34.538627 3933 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0410 16:35:34.548006 3933 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0410 16:35:34.548074 3933 factory.go:537] Starting watch factory I0410 16:35:34.548174 3933 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.548201 3933 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.548496 3933 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.548629 3933 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.548680 3933 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.548763 3933 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.549025 3933 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0410 16:35:34.549146 3933 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.549226 3933 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.549278 3933 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.549302 3933 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.549662 3933 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.549696 3933 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.557487 3933 controller_manager.go:368] Starting the ovnkube controller I0410 16:35:34.557498 3933 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-187.ec2.internal I0410 16:35:34.557547 3933 controller_manager.go:393] NBDB zone sync took: 42.057µs I0410 16:35:34.557558 3933 factory.go:537] Starting watch factory I0410 16:35:34.577786 3933 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.577845 3933 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.577790 3933 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.578120 3933 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.578645 3933 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.579630 3933 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0410 16:35:34.584384 3933 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0410 16:35:34.584423 3933 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0410 16:35:34.589295 3933 ovn_northd.go:121] Registering OVN northd metrics I0410 16:35:34.597920 3933 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0410 16:35:34.603102 3933 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0410 16:35:34.653628 3933 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0410 16:35:34.653642 3933 factory.go:1931] *v1.Namespace informer cache synced successfully I0410 16:35:34.653646 3933 factory.go:1931] *v1.Node informer cache synced successfully I0410 16:35:34.653649 3933 factory.go:1931] *v1.Service informer cache synced successfully I0410 16:35:34.653652 3933 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0410 16:35:34.653655 3933 factory.go:1931] *v1.Pod informer cache synced successfully I0410 16:35:34.653771 3933 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0410 16:35:34.653785 3933 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0410 16:35:34.653793 3933 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0410 16:35:34.653817 3933 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0410 16:35:34.657790 3933 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0410 16:35:34.657809 3933 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0410 16:35:34.659657 3933 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0410 16:35:34.659667 3933 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0410 16:35:34.659708 3933 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.659718 3933 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.662977 3933 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.663936 3933 factory.go:1931] *v1.Pod informer cache synced successfully I0410 16:35:34.663946 3933 factory.go:1931] *v1.NetworkPolicy informer cache synced successfully I0410 16:35:34.663949 3933 factory.go:1931] *v1.Namespace informer cache synced successfully I0410 16:35:34.663952 3933 factory.go:1931] *v1.Node informer cache synced successfully I0410 16:35:34.663955 3933 factory.go:1931] *v1.Service informer cache synced successfully I0410 16:35:34.663957 3933 factory.go:1931] *v1.EndpointSlice informer cache synced successfully I0410 16:35:34.665083 3933 factory.go:1931] *v1.EgressIP informer cache synced successfully I0410 16:35:34.665135 3933 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.665146 3933 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.668345 3933 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.669289 3933 factory.go:1931] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0410 16:35:34.669302 3933 factory.go:1931] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0410 16:35:34.670434 3933 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0410 16:35:34.670492 3933 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.670508 3933 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.673562 3933 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.674503 3933 factory.go:1931] *v1.EgressIP informer cache synced successfully I0410 16:35:34.675585 3933 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0410 16:35:34.675634 3933 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.675645 3933 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.678741 3933 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.679714 3933 factory.go:1931] *v1.EgressFirewall informer cache synced successfully I0410 16:35:34.680855 3933 factory.go:1931] *v1.EgressService informer cache synced successfully I0410 16:35:34.680913 3933 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.680930 3933 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.684624 3933 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.684758 3933 factory.go:1931] *v1.EgressQoS informer cache synced successfully I0410 16:35:34.686590 3933 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0410 16:35:34.686747 3933 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0410 16:35:34.686763 3933 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0410 16:35:34.690257 3933 factory.go:1931] *v1.EgressService informer cache synced successfully I0410 16:35:34.690469 3933 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0410 16:35:34.692450 3933 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0410 16:35:34.692514 3933 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.692533 3933 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.692563 3933 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.692583 3933 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.695522 3933 factory.go:1931] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0410 16:35:34.695830 3933 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.696023 3933 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0410 16:35:34.698109 3933 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0410 16:35:34.698118 3933 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0410 16:35:34.698125 3933 factory.go:649] Watch Factory start up complete, took: 150.053622ms I0410 16:35:34.698200 3933 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0410 16:35:34.700653 3933 factory.go:1931] *v1.NetworkAttachmentDefinition informer cache synced successfully I0410 16:35:34.706365 3933 factory.go:1931] *v1.UserDefinedNetwork informer cache synced successfully I0410 16:35:34.706376 3933 factory.go:1931] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0410 16:35:34.706383 3933 factory.go:649] Watch Factory start up complete, took: 148.82358ms I0410 16:35:34.706389 3933 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-187.ec2.internal" zone I0410 16:35:34.732942 3933 default_node_network_controller.go:709] Initializing the default node network controller I0410 16:35:34.781265 3933 udn_isolation.go:101] Starting UDN host isolation manager I0410 16:35:34.787390 3933 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0410 16:35:34.830964 3933 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0410 16:35:34.831087 3933 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0410 16:35:34.831103 3933 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0410 16:35:34.862206 3933 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0410 16:35:34.862263 3933 default_node_network_controller.go:808] Node ip-10-0-2-187.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0410 16:35:34.867305 3933 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.187"] k8s.ovn.org/zone-name:ip-10-0-2-187.ec2.internal] on node ip-10-0-2-187.ec2.internal I0410 16:35:34.887937 3933 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0410 16:35:34.894048 3933 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0410 16:35:34.894203 3933 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0410 16:35:34.894345 3933 gateway_init.go:263] Preparing Gateway I0410 16:35:34.894354 3933 gateway_shared_intf.go:1658] Creating new gateway I0410 16:35:34.956469 3933 controller_manager.go:427] Waiting for node in zone sync took: 250.068975ms I0410 16:35:34.962435 3933 controller_manager.go:608] Switching to transit router for layer2 networks I0410 16:35:34.962450 3933 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-187.ec2.internal I0410 16:35:34.977182 3933 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0410 16:35:34.978699 3933 iptables.go:108] Creating table: filter chain: FORWARD I0410 16:35:34.983889 3933 controller_manager.go:288] SCTP support detected in OVN I0410 16:35:34.984577 3933 services_controller.go:69] Creating services controller for network=default I0410 16:35:34.985372 3933 udn_enabled_svc.go:80] Waiting for service informer to sync I0410 16:35:34.985556 3933 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0410 16:35:34.985597 3933 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0410 16:35:34.985615 3933 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0410 16:35:34.985626 3933 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0410 16:35:34.985830 3933 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0410 16:35:34.985883 3933 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0410 16:35:34.985927 3933 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 32.982µs I0410 16:35:34.985940 3933 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 56.328µs I0410 16:35:34.985948 3933 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0410 16:35:34.985960 3933 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0410 16:35:34.985971 3933 udn_enabled_svc.go:95] Performing full resync I0410 16:35:34.985997 3933 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 15.048µs I0410 16:35:34.986013 3933 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 3.005µs I0410 16:35:34.985974 3933 default_network_controller.go:321] Starting the default network controller I0410 16:35:34.986033 3933 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 95.535µs I0410 16:35:34.986304 3933 iptables.go:108] Creating table: filter chain: OUTPUT I0410 16:35:34.986397 3933 udn_enabled_svc.go:99] Waiting for handler to sync I0410 16:35:34.986410 3933 udn_enabled_svc.go:104] Starting worker I0410 16:35:34.993858 3933 gateway_shared_intf.go:1787] Gateway Creation Complete I0410 16:35:34.993872 3933 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-187.ec2.internal","mac-address":"0e:1d:2b:95:09:bd","ip-addresses":["10.0.2.187/24"],"ip-address":"10.0.2.187/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:35fdc506-ab56-4d20-b839-7618d7049df8 k8s.ovn.org/node-encap-ips:["10.0.2.187"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.187/24"} k8s.ovn.org/zone-name:ip-10-0-2-187.ec2.internal] on node ip-10-0-2-187.ec2.internal I0410 16:35:34.995893 3933 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0410 16:35:34.995961 3933 repair.go:33] Syncing exgw routes took 49.045µs I0410 16:35:34.995979 3933 default_network_controller.go:407] Starting all the Watchers... I0410 16:35:34.996139 3933 namespace.go:98] [openshift-config] adding namespace I0410 16:35:34.996145 3933 namespace.go:98] [kube-public] adding namespace I0410 16:35:34.996178 3933 namespace.go:98] [openshift-backplane-cse] adding namespace I0410 16:35:34.996180 3933 namespace.go:98] [openshift-host-network] adding namespace I0410 16:35:34.996143 3933 namespace.go:98] [kube-system] adding namespace I0410 16:35:34.996187 3933 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0410 16:35:34.996191 3933 namespace.go:98] [openshift-infra] adding namespace I0410 16:35:34.996200 3933 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0410 16:35:34.996203 3933 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0410 16:35:34.996210 3933 namespace.go:98] [openshift-console] adding namespace I0410 16:35:34.996214 3933 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0410 16:35:34.996219 3933 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0410 16:35:34.996215 3933 namespace.go:98] [openshift-network-operator] adding namespace I0410 16:35:34.996236 3933 namespace.go:98] [openshift-etcd] adding namespace I0410 16:35:34.996295 3933 namespace.go:98] [openshift-apiserver] adding namespace I0410 16:35:34.996781 3933 namespace.go:102] [openshift-config] adding namespace took 629.203µs I0410 16:35:34.996795 3933 namespace.go:98] [openshift-console-operator] adding namespace I0410 16:35:34.997364 3933 namespace.go:102] [openshift-console-operator] adding namespace took 563.989µs I0410 16:35:34.997377 3933 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0410 16:35:34.997845 3933 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 460.757µs I0410 16:35:34.997863 3933 namespace.go:98] [openshift-backplane-csm] adding namespace I0410 16:35:34.998320 3933 namespace.go:102] [kube-public] adding namespace took 2.134917ms I0410 16:35:34.998335 3933 namespace.go:98] [openshift-dns] adding namespace I0410 16:35:34.998902 3933 namespace.go:102] [openshift-backplane-cse] adding namespace took 2.714328ms I0410 16:35:34.998918 3933 namespace.go:98] [openshift-backplane-srep] adding namespace W0410 16:35:34.999536 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-8sxml: pod kube-system/global-pull-secret-syncer-8sxml: no pod IPs found W0410 16:35:34.999548 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-k9zvd: pod kube-system/global-pull-secret-syncer-k9zvd: no pod IPs found I0410 16:35:34.999570 3933 namespace.go:102] [openshift-host-network] adding namespace took 3.3775ms I0410 16:35:34.999586 3933 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0410 16:35:35.000050 3933 namespace.go:102] [kube-system] adding namespace took 3.857253ms I0410 16:35:35.000064 3933 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0410 16:35:35.000615 3933 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 4.417557ms I0410 16:35:35.000630 3933 namespace.go:98] [openshift-console-user-settings] adding namespace I0410 16:35:35.001088 3933 namespace.go:102] [openshift-infra] adding namespace took 4.886253ms I0410 16:35:35.001103 3933 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0410 16:35:35.001684 3933 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.477116ms I0410 16:35:35.001698 3933 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0410 16:35:35.002256 3933 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.045916ms I0410 16:35:35.002273 3933 namespace.go:98] [openshift-machine-api] adding namespace I0410 16:35:35.002779 3933 namespace.go:102] [openshift-console] adding namespace took 6.55457ms I0410 16:35:35.002799 3933 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0410 16:35:35.003696 3933 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.46947ms I0410 16:35:35.003723 3933 namespace.go:98] [openshift-backplane-cee] adding namespace I0410 16:35:35.004181 3933 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.952038ms I0410 16:35:35.004196 3933 namespace.go:98] [openshift-backplane] adding namespace I0410 16:35:35.004717 3933 namespace.go:102] [openshift-network-operator] adding namespace took 8.481311ms I0410 16:35:35.004739 3933 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0410 16:35:35.005187 3933 namespace.go:102] [openshift-etcd] adding namespace took 8.938674ms I0410 16:35:35.005205 3933 namespace.go:98] [openshift-monitoring] adding namespace I0410 16:35:35.005687 3933 namespace.go:102] [openshift-apiserver] adding namespace took 9.383057ms I0410 16:35:35.005703 3933 namespace.go:98] [openshift-authentication] adding namespace I0410 16:35:35.006143 3933 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.27559ms I0410 16:35:35.006208 3933 namespace.go:98] [openshift] adding namespace I0410 16:35:35.006714 3933 namespace.go:102] [openshift-dns] adding namespace took 8.373411ms I0410 16:35:35.006729 3933 namespace.go:98] [openshift-network-diagnostics] adding namespace I0410 16:35:35.007105 3933 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.182284ms I0410 16:35:35.007119 3933 namespace.go:98] [openshift-node] adding namespace I0410 16:35:35.007551 3933 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.958608ms I0410 16:35:35.007566 3933 namespace.go:98] [openshift-kube-apiserver] adding namespace I0410 16:35:35.007958 3933 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.890006ms I0410 16:35:35.007973 3933 namespace.go:98] [default] adding namespace I0410 16:35:35.008566 3933 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.928852ms I0410 16:35:35.008587 3933 namespace.go:98] [kube-node-lease] adding namespace I0410 16:35:35.009070 3933 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.961581ms I0410 16:35:35.009089 3933 namespace.go:98] [openshift-config-managed] adding namespace I0410 16:35:35.009577 3933 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.872043ms I0410 16:35:35.009615 3933 namespace.go:98] [openshift-network-node-identity] adding namespace I0410 16:35:35.010244 3933 namespace.go:102] [openshift-machine-api] adding namespace took 7.96514ms I0410 16:35:35.010261 3933 namespace.go:98] [openshift-network-console] adding namespace I0410 16:35:35.010849 3933 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.044297ms I0410 16:35:35.010864 3933 namespace.go:98] [openshift-customer-monitoring] adding namespace I0410 16:35:35.011472 3933 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.742591ms I0410 16:35:35.011489 3933 namespace.go:98] [dedicated-admin] adding namespace I0410 16:35:35.012134 3933 namespace.go:102] [openshift-backplane] adding namespace took 7.929799ms I0410 16:35:35.012178 3933 namespace.go:98] [openshift-config-operator] adding namespace I0410 16:35:35.012750 3933 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.003529ms I0410 16:35:35.012764 3933 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0410 16:35:35.013322 3933 namespace.go:102] [openshift-monitoring] adding namespace took 8.109057ms I0410 16:35:35.013341 3933 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0410 16:35:35.013849 3933 namespace.go:102] [openshift-authentication] adding namespace took 8.137632ms I0410 16:35:35.013870 3933 namespace.go:98] [openshift-authentication-operator] adding namespace W0410 16:35:35.014486 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-6dd2r: pod openshift-network-diagnostics/network-check-target-6dd2r: no pod IPs found I0410 16:35:35.014490 3933 namespace.go:102] [openshift] adding namespace took 8.272315ms I0410 16:35:35.014491 3933 default_node_network_controller.go:518] Node connection status = connected I0410 16:35:35.014509 3933 namespace.go:98] [openshift-apiserver-operator] adding namespace W0410 16:35:35.014500 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-c868t: pod openshift-network-diagnostics/network-check-target-c868t: no pod IPs found W0410 16:35:35.014539 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-ch6vc: pod openshift-network-diagnostics/network-check-target-ch6vc: no pod IPs found I0410 16:35:35.015091 3933 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.355051ms I0410 16:35:35.015109 3933 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0410 16:35:35.015572 3933 namespace.go:102] [openshift-node] adding namespace took 8.446747ms I0410 16:35:35.015584 3933 namespace.go:98] [openshift-service-ca-operator] adding namespace I0410 16:35:35.015998 3933 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.42654ms I0410 16:35:35.016013 3933 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0410 16:35:35.016435 3933 namespace.go:102] [default] adding namespace took 8.456988ms I0410 16:35:35.016448 3933 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0410 16:35:35.016854 3933 namespace.go:102] [kube-node-lease] adding namespace took 8.260734ms I0410 16:35:35.016867 3933 namespace.go:98] [openshift-kube-scheduler] adding namespace I0410 16:35:35.017266 3933 namespace.go:102] [openshift-config-managed] adding namespace took 8.17162ms I0410 16:35:35.017279 3933 namespace.go:98] [openshift-multus] adding namespace I0410 16:35:35.017895 3933 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.271424ms I0410 16:35:35.017913 3933 namespace.go:98] [openshift-backplane-mobb] adding namespace I0410 16:35:35.018352 3933 namespace.go:102] [openshift-network-console] adding namespace took 8.085471ms I0410 16:35:35.018374 3933 namespace.go:98] [openshift-cluster-version] adding namespace I0410 16:35:35.018837 3933 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.966193ms I0410 16:35:35.018851 3933 namespace.go:98] [openshift-package-operator] adding namespace I0410 16:35:35.019361 3933 namespace.go:102] [dedicated-admin] adding namespace took 7.86666ms I0410 16:35:35.019375 3933 namespace.go:98] [openshift-controller-manager] adding namespace I0410 16:35:35.019776 3933 namespace.go:102] [openshift-config-operator] adding namespace took 7.591291ms I0410 16:35:35.019788 3933 namespace.go:98] [openshift-backplane-tam] adding namespace I0410 16:35:35.020268 3933 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.49946ms I0410 16:35:35.020279 3933 namespace.go:98] [openshift-route-controller-manager] adding namespace I0410 16:35:35.020808 3933 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.458817ms I0410 16:35:35.020828 3933 namespace.go:98] [openshift-ingress-canary] adding namespace I0410 16:35:35.021359 3933 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.482611ms I0410 16:35:35.021373 3933 namespace.go:98] [openshift-dns-operator] adding namespace I0410 16:35:35.021937 3933 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.414044ms I0410 16:35:35.022590 3933 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.475914ms I0410 16:35:35.022602 3933 namespace.go:98] [openshift-marketplace] adding namespace I0410 16:35:35.023125 3933 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.53616ms I0410 16:35:35.023137 3933 namespace.go:98] [openshift-must-gather-operator] adding namespace I0410 16:35:35.024072 3933 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.053495ms I0410 16:35:35.024084 3933 namespace.go:98] [openshift-ingress-operator] adding namespace I0410 16:35:35.024611 3933 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.158091ms I0410 16:35:35.024621 3933 namespace.go:98] [openshift-operators] adding namespace W0410 16:35:35.025080 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-7lq8s: pod openshift-multus/network-metrics-daemon-7lq8s: no pod IPs found W0410 16:35:35.025092 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-rh8mc: pod openshift-multus/network-metrics-daemon-rh8mc: no pod IPs found W0410 16:35:35.025098 3933 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-vfw65: pod openshift-multus/network-metrics-daemon-vfw65: no pod IPs found I0410 16:35:35.025118 3933 namespace.go:102] [openshift-kube-scheduler] adding namespace took 8.247115ms I0410 16:35:35.025128 3933 namespace.go:98] [openshift-insights] adding namespace I0410 16:35:35.025705 3933 namespace.go:102] [openshift-multus] adding namespace took 8.422084ms I0410 16:35:35.025716 3933 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0410 16:35:35.026244 3933 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.323503ms I0410 16:35:35.026260 3933 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0410 16:35:35.026849 3933 namespace.go:102] [openshift-cluster-version] adding namespace took 8.467133ms I0410 16:35:35.026868 3933 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0410 16:35:35.027409 3933 namespace.go:102] [openshift-package-operator] adding namespace took 8.552011ms I0410 16:35:35.027426 3933 namespace.go:98] [openshift-operators-redhat] adding namespace I0410 16:35:35.027972 3933 namespace.go:102] [openshift-controller-manager] adding namespace took 8.591605ms I0410 16:35:35.027987 3933 namespace.go:98] [openshift-machine-config-operator] adding namespace I0410 16:35:35.028440 3933 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.6463ms I0410 16:35:35.028458 3933 namespace.go:98] [openshift-image-registry] adding namespace I0410 16:35:35.028958 3933 namespace.go:102] [openshift-route-controller-manager] adding namespace took 8.670877ms I0410 16:35:35.028977 3933 namespace.go:98] [openshift-logging] adding namespace I0410 16:35:35.029505 3933 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.667472ms I0410 16:35:35.029522 3933 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0410 16:35:35.030071 3933 namespace.go:102] [openshift-dns-operator] adding namespace took 8.693285ms I0410 16:35:35.030082 3933 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0410 16:35:35.030675 3933 namespace.go:102] [openshift-marketplace] adding namespace took 8.066058ms I0410 16:35:35.030693 3933 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0410 16:35:35.031180 3933 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.036237ms I0410 16:35:35.031628 3933 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.539276ms I0410 16:35:35.032031 3933 namespace.go:102] [openshift-operators] adding namespace took 7.404447ms I0410 16:35:35.032445 3933 namespace.go:102] [openshift-insights] adding namespace took 7.311202ms I0410 16:35:35.032880 3933 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.160398ms I0410 16:35:35.033506 3933 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.239577ms I0410 16:35:35.034012 3933 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.136943ms I0410 16:35:35.034025 3933 namespace.go:98] [openshift-ingress] adding namespace I0410 16:35:35.034526 3933 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.095675ms I0410 16:35:35.035065 3933 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.072595ms I0410 16:35:35.035558 3933 namespace.go:102] [openshift-image-registry] adding namespace took 7.093293ms I0410 16:35:35.035966 3933 namespace.go:102] [openshift-logging] adding namespace took 6.982903ms I0410 16:35:35.035980 3933 namespace.go:98] [open-cluster-management-2pjda9eb2v9960j6pvh7mvgjb27ifi1u] adding namespace I0410 16:35:35.036430 3933 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.901473ms I0410 16:35:35.036971 3933 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.884366ms I0410 16:35:35.037474 3933 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.774602ms I0410 16:35:35.037895 3933 namespace.go:102] [openshift-ingress] adding namespace took 3.863453ms I0410 16:35:35.038425 3933 namespace.go:102] [open-cluster-management-2pjda9eb2v9960j6pvh7mvgjb27ifi1u] adding namespace took 2.439854ms I0410 16:35:35.038834 3933 zone_ic_handler.go:186] Time taken to create transit switch: 307.981µs I0410 16:35:35.038924 3933 master.go:532] Adding or Updating local node "ip-10-0-2-187.ec2.internal" for network "default" I0410 16:35:35.038948 3933 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-66.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-66.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-66.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-66.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-66.ec2.internal I0410 16:35:35.038975 3933 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-177.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-177.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-177.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-177.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-177.ec2.internal W0410 16:35:35.046526 3933 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-vfw65: pod openshift-multus/network-metrics-daemon-vfw65: no pod IPs found W0410 16:35:35.046541 3933 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-8sxml: pod kube-system/global-pull-secret-syncer-8sxml: no pod IPs found W0410 16:35:35.046559 3933 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-ch6vc: pod openshift-network-diagnostics/network-check-target-ch6vc: no pod IPs found I0410 16:35:35.047477 3933 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-187.ec2.internal for the network default I0410 16:35:35.048567 3933 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0410 16:35:35.048589 3933 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0410 16:35:35.048602 3933 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0410 16:35:35.048625 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-66.ec2.internal in network "default" I0410 16:35:35.048650 3933 node_tracker.go:235] Node ip-10-0-0-66.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-66.ec2.internal" I0410 16:35:35.048661 3933 node_tracker.go:169] Node ip-10-0-0-66.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:35.048669 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:35.048675 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-177.ec2.internal in network "default" I0410 16:35:35.048682 3933 node_tracker.go:235] Node ip-10-0-1-177.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-177.ec2.internal" I0410 16:35:35.048687 3933 node_tracker.go:169] Node ip-10-0-1-177.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:35.048691 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:35.048694 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-187.ec2.internal in network "default" I0410 16:35:35.048717 3933 node_tracker.go:169] Node ip-10-0-2-187.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:35.048724 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:35.148943 3933 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0410 16:35:35.148966 3933 services_controller.go:208] Setting up event handlers for services for network=default I0410 16:35:35.149027 3933 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0410 16:35:35.149086 3933 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0410 16:35:35.149113 3933 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0410 16:35:35.250116 3933 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0410 16:35:35.250136 3933 repair.go:60] Starting repairing loop for services I0410 16:35:35.250232 3933 repair.go:131] Deleted 0 stale service LBs I0410 16:35:35.250251 3933 repair.go:137] Deleted 0 stale Chassis Template Vars I0410 16:35:35.250268 3933 repair.go:62] Finished repairing loop for services: 134.549µs I0410 16:35:35.250294 3933 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0410 16:35:35.250303 3933 services_controller.go:254] Starting workers for network=default I0410 16:35:35.250573 3933 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-6dd2r, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-6dd2r: suppressed error logged: pod openshift-network-diagnostics/network-check-target-6dd2r: no pod IPs found I0410 16:35:35.250591 3933 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-ch6vc] creating logical port openshift-network-diagnostics_network-check-target-ch6vc for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:35.250612 3933 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-vfw65] creating logical port openshift-multus_network-metrics-daemon-vfw65 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:35.250626 3933 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-k9zvd, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-k9zvd: suppressed error logged: pod kube-system/global-pull-secret-syncer-k9zvd: no pod IPs found I0410 16:35:35.250635 3933 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-rh8mc, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-rh8mc: suppressed error logged: pod openshift-multus/network-metrics-daemon-rh8mc: no pod IPs found I0410 16:35:35.250726 3933 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-7lq8s, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-7lq8s: suppressed error logged: pod openshift-multus/network-metrics-daemon-7lq8s: no pod IPs found I0410 16:35:35.250747 3933 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-ch6vc I0410 16:35:35.250765 3933 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-vfw65 I0410 16:35:35.266977 3933 pod.go:62] [openshift-network-diagnostics/network-check-target-ch6vc] pod update took 16.306254ms I0410 16:35:35.267015 3933 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-ch6vc] addLogicalPort annotation time took 16.333317ms I0410 16:35:35.267316 3933 pod.go:62] [openshift-multus/network-metrics-daemon-vfw65] pod update took 16.640535ms I0410 16:35:35.267334 3933 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-vfw65] addLogicalPort annotation time took 16.661525ms I0410 16:35:35.267851 3933 pods.go:271] [openshift-network-diagnostics/network-check-target-ch6vc] addLogicalPort took 17.269329ms, libovsdb time 606.486µs I0410 16:35:35.268147 3933 pods.go:271] [openshift-multus/network-metrics-daemon-vfw65] addLogicalPort took 17.54133ms, libovsdb time 634.532µs I0410 16:35:35.268200 3933 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-c868t, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-c868t: suppressed error logged: pod openshift-network-diagnostics/network-check-target-c868t: no pod IPs found I0410 16:35:35.268220 3933 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-8sxml] creating logical port kube-system_global-pull-secret-syncer-8sxml for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:35.268266 3933 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-8sxml I0410 16:35:35.281667 3933 pod.go:62] [kube-system/global-pull-secret-syncer-8sxml] pod update took 13.418708ms I0410 16:35:35.281683 3933 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-8sxml] addLogicalPort annotation time took 13.436544ms I0410 16:35:35.282500 3933 pods.go:271] [kube-system/global-pull-secret-syncer-8sxml] addLogicalPort took 14.285556ms, libovsdb time 619.568µs I0410 16:35:35.282922 3933 admin_network_policy_controller.go:227] Starting controller default-network-controller I0410 16:35:35.282940 3933 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0410 16:35:35.282951 3933 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0410 16:35:35.282955 3933 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0410 16:35:35.283188 3933 repair.go:29] Repairing admin network policies took 225.084µs I0410 16:35:35.283290 3933 repair.go:92] Repairing baseline admin network policies took 93.808µs I0410 16:35:35.283722 3933 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0410 16:35:35.283737 3933 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0410 16:35:35.285639 3933 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0410 16:35:35.285661 3933 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0410 16:35:35.285673 3933 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0410 16:35:35.286746 3933 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0410 16:35:35.286765 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0410 16:35:35.286774 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0410 16:35:35.288407 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0410 16:35:35.288425 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0410 16:35:35.288434 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0410 16:35:35.288960 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0410 16:35:35.288981 3933 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0410 16:35:35.288997 3933 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0410 16:35:35.291051 3933 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0410 16:35:35.291071 3933 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0410 16:35:35.291087 3933 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0410 16:35:35.293017 3933 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0410 16:35:35.293034 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0410 16:35:35.293041 3933 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0410 16:35:35.294580 3933 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0410 16:35:35.294603 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0410 16:35:35.294614 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0410 16:35:35.296224 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0410 16:35:35.296246 3933 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0410 16:35:35.296254 3933 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0410 16:35:35.296549 3933 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0410 16:35:35.296567 3933 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0410 16:35:35.296576 3933 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0410 16:35:35.297672 3933 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0410 16:35:35.297699 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0410 16:35:35.297710 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0410 16:35:35.298236 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0410 16:35:35.298254 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0410 16:35:35.298262 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0410 16:35:35.299733 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0410 16:35:35.299749 3933 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0410 16:35:35.299757 3933 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0410 16:35:35.300242 3933 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0410 16:35:35.300266 3933 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0410 16:35:35.300279 3933 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0410 16:35:35.301218 3933 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0410 16:35:35.301236 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0410 16:35:35.301643 3933 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0410 16:35:35.301654 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0410 16:35:35.302543 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0410 16:35:35.302569 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0410 16:35:35.302584 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0410 16:35:35.303103 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0410 16:35:35.303123 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0410 16:35:35.303132 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0410 16:35:35.303546 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0410 16:35:35.303564 3933 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0410 16:35:35.303579 3933 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0410 16:35:35.304787 3933 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0410 16:35:35.304805 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0410 16:35:35.304849 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0410 16:35:35.305662 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0410 16:35:35.305687 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0410 16:35:35.305698 3933 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0410 16:35:35.306462 3933 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0410 16:35:35.306484 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0410 16:35:35.306490 3933 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0410 16:35:35.306808 3933 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0410 16:35:35.306827 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0410 16:35:35.306835 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0410 16:35:35.307449 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0410 16:35:35.307467 3933 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0410 16:35:35.307475 3933 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0410 16:35:35.308668 3933 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0410 16:35:35.308684 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0410 16:35:35.308691 3933 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0410 16:35:35.309241 3933 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0410 16:35:35.309280 3933 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0410 16:35:35.309290 3933 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0410 16:35:35.309816 3933 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0410 16:35:35.309839 3933 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0410 16:35:35.309849 3933 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0410 16:35:35.311067 3933 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0410 16:35:35.311090 3933 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0410 16:35:35.311101 3933 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0410 16:35:35.313123 3933 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0410 16:35:35.313144 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0410 16:35:35.313171 3933 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0410 16:35:35.313670 3933 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0410 16:35:35.313691 3933 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0410 16:35:35.313707 3933 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0410 16:35:35.314208 3933 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0410 16:35:35.318969 3933 egressfirewall.go:319] Starting EgressFirewall controller I0410 16:35:35.319001 3933 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0410 16:35:35.319013 3933 controller.go:133] Adding controller egress-firewall-controller event handlers I0410 16:35:35.319031 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0410 16:35:35.319037 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0410 16:35:35.319049 3933 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0410 16:35:35.319051 3933 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0410 16:35:35.319380 3933 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0410 16:35:35.319397 3933 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0410 16:35:35.319409 3933 egressqos.go:193] Setting up event handlers for EgressQoS I0410 16:35:35.319495 3933 egressfirewall.go:628] Syncing node "ip-10-0-1-177.ec2.internal" for egress firewall I0410 16:35:35.319511 3933 egressfirewall.go:628] Syncing node "ip-10-0-2-187.ec2.internal" for egress firewall I0410 16:35:35.319515 3933 egressfirewall.go:628] Syncing node "ip-10-0-0-66.ec2.internal" for egress firewall I0410 16:35:35.319592 3933 egressqos.go:245] Starting EgressQoS Controller I0410 16:35:35.319607 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0410 16:35:35.319622 3933 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0410 16:35:35.319632 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0410 16:35:35.319642 3933 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0410 16:35:35.319650 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0410 16:35:35.319656 3933 shared_informer.go:356] "Caches are synced" controller="egressqos" I0410 16:35:35.319662 3933 egressqos.go:259] Repairing EgressQoSes I0410 16:35:35.319668 3933 egressqos.go:399] Starting repairing loop for egressqos I0410 16:35:35.319760 3933 egressqos.go:401] Finished repairing loop for egressqos: 92.924µs I0410 16:35:35.319821 3933 egressservice_zone.go:132] Setting up event handlers for Egress Services I0410 16:35:35.319867 3933 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-187.ec2.internal I0410 16:35:35.319916 3933 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-187.ec2.internal : 48.936µs I0410 16:35:35.320025 3933 egressservice_zone.go:210] Starting Egress Services Controller I0410 16:35:35.320041 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0410 16:35:35.320050 3933 shared_informer.go:356] "Caches are synced" controller="egressservices" I0410 16:35:35.320058 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0410 16:35:35.320064 3933 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0410 16:35:35.320073 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0410 16:35:35.320080 3933 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0410 16:35:35.320087 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0410 16:35:35.320094 3933 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0410 16:35:35.320099 3933 egressservice_zone.go:228] Repairing Egress Services I0410 16:35:35.320390 3933 master_controller.go:88] Starting Admin Policy Based Route Controller I0410 16:35:35.320399 3933 external_controller.go:278] Starting Admin Policy Based Route Controller I0410 16:35:35.320464 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-66.ec2.internal I0410 16:35:35.320474 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-66.ec2.internal: 11.416µs I0410 16:35:35.320480 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-177.ec2.internal I0410 16:35:35.320484 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-177.ec2.internal: 3.748µs I0410 16:35:35.320488 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-187.ec2.internal I0410 16:35:35.320493 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-187.ec2.internal: 4.962µs I0410 16:35:35.320515 3933 default_network_controller.go:568] Completing all the Watchers took 324.527551ms I0410 16:35:35.320531 3933 default_network_controller.go:572] Starting unidling controllers I0410 16:35:35.320540 3933 unidle.go:46] Registering OVN SB ControllerEvent handler I0410 16:35:35.320548 3933 unidle.go:63] Populating Initial ContollerEvent events I0410 16:35:35.320562 3933 unidle.go:79] Setting up event handlers for services I0410 16:35:35.320623 3933 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0410 16:35:35.514482 3933 default_node_network_controller.go:518] Node connection status = connected I0410 16:35:35.946649 3933 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0410 16:35:36.014747 3933 default_node_network_controller.go:518] Node connection status = connected I0410 16:35:36.032758 3933 bridgeconfig.go:537] Gateway is ready I0410 16:35:36.032777 3933 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0410 16:35:36.044925 3933 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.187. Updating OVN encap IP. I0410 16:35:36.050757 3933 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.187 - it is already configured I0410 16:35:36.050771 3933 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.187/24:{} 172.20.0.1/32:{}]. Updating annotations. I0410 16:35:36.051008 3933 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.187/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-187.ec2.internal","mac-address":"0e:1d:2b:95:09:bd","ip-addresses":["10.0.2.187/24"],"ip-address":"10.0.2.187/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:35fdc506-ab56-4d20-b839-7618d7049df8 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.187/24"}] on node ip-10-0-2-187.ec2.internal I0410 16:35:36.066610 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-187.ec2.internal I0410 16:35:36.066631 3933 egressfirewall.go:628] Syncing node "ip-10-0-2-187.ec2.internal" for egress firewall I0410 16:35:36.066636 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-187.ec2.internal: 29.755µs I0410 16:35:36.066660 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-187.ec2.internal in network "default" I0410 16:35:36.066699 3933 master.go:532] Adding or Updating local node "ip-10-0-2-187.ec2.internal" for network "default" I0410 16:35:36.066712 3933 node_tracker.go:169] Node ip-10-0-2-187.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:36.066725 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:36.067862 3933 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.187 I0410 16:35:36.067920 3933 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-187.ec2.internal I0410 16:35:36.072017 3933 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-187.ec2.internal. Updating no re-route policies I0410 16:35:36.072104 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-t4ff8 I0410 16:35:36.072117 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-vfw65 I0410 16:35:36.072117 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-2gcbk I0410 16:35:36.072132 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s4skl I0410 16:35:36.072137 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-2gcbk I0410 16:35:36.072137 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-smqm4 I0410 16:35:36.072144 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s4skl I0410 16:35:36.072146 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-5hnjb I0410 16:35:36.072144 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-187.ec2.internal I0410 16:35:36.072172 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-5hnjb I0410 16:35:36.072173 3933 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-52hhc I0410 16:35:36.072176 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-187.ec2.internal I0410 16:35:36.072172 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-smqm4 I0410 16:35:36.072186 3933 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-52hhc I0410 16:35:36.072188 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-s4skl after 0 failed attempt(s) I0410 16:35:36.072191 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-2gcbk after 0 failed attempt(s) I0410 16:35:36.072120 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-ch6vc I0410 16:35:36.072194 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-5hnjb after 0 failed attempt(s) I0410 16:35:36.072196 3933 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-bx5pl I0410 16:35:36.072204 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-cvkvw I0410 16:35:36.072203 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-ch6vc I0410 16:35:36.072214 3933 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-bx5pl I0410 16:35:36.072219 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-cvkvw I0410 16:35:36.072229 3933 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-187.ec2.internal I0410 16:35:36.072230 3933 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-bx5pl after 0 failed attempt(s) I0410 16:35:36.072240 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-cvkvw after 0 failed attempt(s) I0410 16:35:36.072141 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-6wqgv I0410 16:35:36.072245 3933 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-187.ec2.internal I0410 16:35:36.072250 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-6wqgv I0410 16:35:36.072281 3933 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-187.ec2.internal after 0 failed attempt(s) I0410 16:35:36.072239 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-187.ec2.internal after 0 failed attempt(s) I0410 16:35:36.072133 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-vfw65 I0410 16:35:36.072319 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-6wqgv after 0 failed attempt(s) I0410 16:35:36.072326 3933 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-vfw65] creating logical port openshift-multus_network-metrics-daemon-vfw65 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:36.072125 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-t4ff8 I0410 16:35:36.072381 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-t4ff8 after 0 failed attempt(s) I0410 16:35:36.072106 3933 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-hcjw9 I0410 16:35:36.072401 3933 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-hcjw9 I0410 16:35:36.072419 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-hcjw9 after 0 failed attempt(s) I0410 16:35:36.072203 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-smqm4 after 0 failed attempt(s) I0410 16:35:36.072107 3933 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-8sxml I0410 16:35:36.072436 3933 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-8sxml I0410 16:35:36.072464 3933 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-8sxml] creating logical port kube-system_global-pull-secret-syncer-8sxml for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:36.072207 3933 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-52hhc after 0 failed attempt(s) I0410 16:35:36.072242 3933 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-ch6vc] creating logical port openshift-network-diagnostics_network-check-target-ch6vc for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:36.072956 3933 pods.go:271] [openshift-multus/network-metrics-daemon-vfw65] addLogicalPort took 637.261µs, libovsdb time 230.582µs I0410 16:35:36.072970 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-vfw65 after 0 failed attempt(s) I0410 16:35:36.073244 3933 pods.go:271] [openshift-network-diagnostics/network-check-target-ch6vc] addLogicalPort took 1.015287ms, libovsdb time 465.093µs I0410 16:35:36.073257 3933 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-ch6vc after 0 failed attempt(s) I0410 16:35:36.073328 3933 pods.go:271] [kube-system/global-pull-secret-syncer-8sxml] addLogicalPort took 869.127µs, libovsdb time 388.803µs I0410 16:35:36.073344 3933 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-8sxml after 0 failed attempt(s) I0410 16:35:36.083386 3933 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0410 16:35:36.096340 3933 iptables.go:108] Creating table: mangle chain: OUTPUT I0410 16:35:36.099864 3933 iptables.go:108] Creating table: nat chain: OUTPUT I0410 16:35:36.120112 3933 iptables.go:108] Creating table: nat chain: PREROUTING I0410 16:35:36.163837 3933 iptables.go:108] Creating table: filter chain: FORWARD I0410 16:35:36.183181 3933 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0410 16:35:36.183337 3933 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0410 16:35:36.183404 3933 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0410 16:35:36.183517 3933 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0410 16:35:36.183593 3933 gateway_init.go:325] Gateway and management port readiness took 1.174202065s I0410 16:35:36.183621 3933 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0410 16:35:36.183656 3933 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0410 16:35:36.183669 3933 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0410 16:35:36.183678 3933 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0410 16:35:36.183730 3933 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0410 16:35:36.183814 3933 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.824µs I0410 16:35:36.184075 3933 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0410 16:35:36.184106 3933 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 15.756µs I0410 16:35:36.184114 3933 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 41.213µs I0410 16:35:36.184120 3933 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0410 16:35:36.184130 3933 nad_controller.go:173] [node-nad-controller NAD controller]: started I0410 16:35:36.184134 3933 default_node_network_controller.go:898] Starting the default node network controller I0410 16:35:36.184139 3933 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0410 16:35:36.184183 3933 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.342µs I0410 16:35:36.184330 3933 default_node_network_controller.go:1449] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0410 16:35:36.184339 3933 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0410 16:35:36.184364 3933 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000af4880 0xc0002d1e30 0xc0002d1ea0 0xc0002d1f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc000c82d70 0xc0002ec150 0xc0002ec1c0 0xc000c3e370 0xc0061df2d0 0xc0002ec230 map[0x2f8f0c0:0xc00251ad80 0x2f8f600:0xc00251af00 0x2f90080:0xc00251acc0 0x2f90320:0xc00251ad20 0x2f90b00:0xc00251ade0 0x2f90da0:0xc00251ae40 0x2f91040:0xc00251aea0 0x2f91820:0xc00251b1a0 0x2f91ac0:0xc00251b200 0x2fbc3e0:0xc00110f560 0x2fbc780:0xc00110f5c0 0x2fbd260:0xc000ccc4e0 0x2fbebc0:0xc001ba2000 0x2fbf6a0:0xc002610ea0 0x2fc0520:0xc000f5a6c0] 0xc0004d6fc0 0} I0410 16:35:36.184424 3933 gateway.go:151] Starting gateway service sync I0410 16:35:36.184863 3933 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0410 16:35:36.187295 3933 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0410 16:35:36.196887 3933 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0410 16:35:36.198115 3933 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0410 16:35:36.199382 3933 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0410 16:35:36.235035 3933 gateway.go:168] Gateway service sync done. Time taken: 50.598794ms I0410 16:35:36.235678 3933 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32423 I0410 16:35:36.235777 3933 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000af4880 0xc0002d1e30 0xc0002d1ea0 0xc0002d1f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc000c82d70 0xc0002ec150 0xc0002ec1c0 0xc000c3e370 0xc0061df2d0 0xc0002ec230 map[0x2f8f0c0:0xc00251ad80 0x2f8f600:0xc00251af00 0x2f90080:0xc00251acc0 0x2f90320:0xc00251ad20 0x2f90b00:0xc00251ade0 0x2f90da0:0xc00251ae40 0x2f91040:0xc00251aea0 0x2f91820:0xc00251b1a0 0x2f91ac0:0xc00251b200 0x2fbc3e0:0xc00110f560 0x2fbc780:0xc00110f5c0 0x2fbd260:0xc000ccc4e0 0x2fbebc0:0xc001ba2000 0x2fbf6a0:0xc002610ea0 0x2fc0520:0xc000f5a6c0] 0xc0004d6fc0 0} I0410 16:35:36.236257 3933 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000af4880 0xc0002d1e30 0xc0002d1ea0 0xc0002d1f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc000c82d70 0xc0002ec150 0xc0002ec1c0 0xc000c3e370 0xc0061df2d0 0xc0002ec230 map[0x2f8f0c0:0xc00251ad80 0x2f8f600:0xc00251af00 0x2f90080:0xc00251acc0 0x2f90320:0xc00251ad20 0x2f90b00:0xc00251ade0 0x2f90da0:0xc00251ae40 0x2f91040:0xc00251aea0 0x2f91820:0xc00251b1a0 0x2f91ac0:0xc00251b200 0x2fbc3e0:0xc00110f560 0x2fbc780:0xc00110f5c0 0x2fbd260:0xc000ccc4e0 0x2fbebc0:0xc001ba2000 0x2fbf6a0:0xc002610ea0 0x2fc0520:0xc000f5a6c0] 0xc0004d6fc0 0} I0410 16:35:36.236294 3933 gateway.go:519] Reconciling gateway with updates I0410 16:35:36.241398 3933 gateway.go:326] Spawning Conntrack Rule Check Thread I0410 16:35:36.241487 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0410 16:35:36.241487 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0410 16:35:36.241501 3933 default_node_network_controller.go:943] Gateway and management port readiness took 57.152687ms I0410 16:35:36.241504 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0410 16:35:36.241511 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0410 16:35:36.241509 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0410 16:35:36.241521 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0410 16:35:36.241520 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0410 16:35:36.241518 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0410 16:35:36.241532 3933 port_claim.go:71] Svc openshift-ingress/router-default: port 31521 is already open, no action needed I0410 16:35:36.241528 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0410 16:35:36.241539 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0410 16:35:36.241541 3933 port_claim.go:71] Svc openshift-ingress/router-default: port 31262 is already open, no action needed I0410 16:35:36.241539 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0410 16:35:36.241545 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0410 16:35:36.241549 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0410 16:35:36.241564 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0410 16:35:36.241566 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0410 16:35:36.241591 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0410 16:35:36.241596 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0410 16:35:36.241604 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0410 16:35:36.241606 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0410 16:35:36.241550 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0410 16:35:36.241607 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0410 16:35:36.241659 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0410 16:35:36.241671 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0410 16:35:36.241674 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0410 16:35:36.241674 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0410 16:35:36.241665 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0410 16:35:36.241683 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0410 16:35:36.241687 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0410 16:35:36.241696 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0410 16:35:36.241696 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0410 16:35:36.241708 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0410 16:35:36.241715 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0410 16:35:36.241729 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0410 16:35:36.241730 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0410 16:35:36.241730 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0410 16:35:36.241732 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0410 16:35:36.241734 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0410 16:35:36.241743 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0410 16:35:36.241745 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0410 16:35:36.241747 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0410 16:35:36.241749 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0410 16:35:36.241746 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0410 16:35:36.241758 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0410 16:35:36.241762 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0410 16:35:36.241764 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0410 16:35:36.241770 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0410 16:35:36.241770 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0410 16:35:36.241754 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0410 16:35:36.241780 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0410 16:35:36.241738 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0410 16:35:36.241792 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0410 16:35:36.241721 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0410 16:35:36.241844 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0410 16:35:36.241899 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0410 16:35:36.241917 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0410 16:35:36.241918 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0410 16:35:36.241924 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0410 16:35:36.241927 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0410 16:35:36.241926 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0410 16:35:36.241931 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0410 16:35:36.241933 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0410 16:35:36.241942 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0410 16:35:36.241945 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0410 16:35:36.241949 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0410 16:35:36.241951 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0410 16:35:36.241948 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0410 16:35:36.241958 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0410 16:35:36.241960 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0410 16:35:36.241963 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0410 16:35:36.241972 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0410 16:35:36.241975 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0410 16:35:36.241979 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0410 16:35:36.241982 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0410 16:35:36.241985 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0410 16:35:36.241989 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0410 16:35:36.241993 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0410 16:35:36.241991 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0410 16:35:36.241999 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0410 16:35:36.241997 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0410 16:35:36.242001 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0410 16:35:36.242003 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0410 16:35:36.242007 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0410 16:35:36.242008 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0410 16:35:36.242013 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0410 16:35:36.241979 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0410 16:35:36.242016 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0410 16:35:36.242022 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0410 16:35:36.242029 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0410 16:35:36.241983 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0410 16:35:36.242030 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0410 16:35:36.241963 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0410 16:35:36.242041 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0410 16:35:36.242050 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0410 16:35:36.242050 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0410 16:35:36.241992 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0410 16:35:36.242063 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0410 16:35:36.242011 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0410 16:35:36.242033 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0410 16:35:36.242211 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0410 16:35:36.242111 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0410 16:35:36.243043 3933 node_ip_handler_linux.go:160] Node IP manager is running W0410 16:35:36.280899 3933 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0410 16:35:36.281058 3933 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0410 16:35:36.281266 3933 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0410 16:35:36.289233 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0410 16:35:36.289242 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0410 16:35:36.289258 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0410 16:35:36.289265 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0410 16:35:36.289271 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0410 16:35:36.289278 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0410 16:35:36.289284 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0410 16:35:36.289290 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0410 16:35:36.289296 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0410 16:35:36.289302 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0410 16:35:36.289307 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0410 16:35:36.327877 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0410 16:35:36.327907 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0410 16:35:36.327917 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0410 16:35:36.327929 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0410 16:35:36.327939 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0410 16:35:36.327948 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0410 16:35:36.327959 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0410 16:35:36.327968 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0410 16:35:36.327977 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0410 16:35:36.327989 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0410 16:35:36.328000 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0410 16:35:36.328011 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0410 16:35:36.366367 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0410 16:35:36.366388 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0410 16:35:36.366398 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0410 16:35:36.366404 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0410 16:35:36.366412 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0410 16:35:36.366418 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0410 16:35:36.366424 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0410 16:35:36.366430 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0410 16:35:36.366436 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) W0410 16:35:37.971959 3933 egressip_healthcheck.go:76] Health checking using insecure connection I0410 16:35:37.972025 3933 default_node_network_controller.go:1369] Starting node controller node sync I0410 16:35:37.972041 3933 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0410 16:35:37.983731 3933 default_node_network_controller.go:1411] Node controller node sync done. Time taken: 11.689972ms I0410 16:35:37.983893 3933 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0410 16:35:37.984170 3933 egressservice_node.go:93] Setting up event handlers for Egress Services I0410 16:35:37.984363 3933 egressservice_node.go:183] Starting Egress Services Controller I0410 16:35:37.984383 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0410 16:35:37.984392 3933 shared_informer.go:356] "Caches are synced" controller="egressservices" I0410 16:35:37.984397 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0410 16:35:37.984402 3933 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0410 16:35:37.984407 3933 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0410 16:35:37.984412 3933 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0410 16:35:37.984416 3933 egressservice_node.go:197] Repairing Egress Services I0410 16:35:37.995448 3933 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0410 16:35:37.995458 3933 external_controller.go:278] Starting Admin Policy Based Route Controller I0410 16:35:37.995598 3933 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0410 16:35:37.995608 3933 link_network_manager.go:117] Link manager is running I0410 16:35:37.995616 3933 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0410 16:35:37.995665 3933 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0410 16:35:37.996277 3933 vrf_manager.go:128] VRF manager is running I0410 16:35:37.996595 3933 node_controller_manager.go:437] Removing flows to drop GARP I0410 16:35:37.996604 3933 gateway.go:519] Reconciling gateway with updates I0410 16:35:38.020274 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0410 16:35:38.020283 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0410 16:35:38.020303 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0410 16:35:38.020306 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0410 16:35:38.020302 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0410 16:35:38.020316 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0410 16:35:38.020327 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0410 16:35:38.020338 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0410 16:35:38.020350 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0410 16:35:38.020275 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0410 16:35:38.020377 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0410 16:35:38.020380 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0410 16:35:38.020379 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0410 16:35:38.020378 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0410 16:35:38.020394 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0410 16:35:38.020395 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0410 16:35:38.020397 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0410 16:35:38.020402 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0410 16:35:38.020408 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0410 16:35:38.020411 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0410 16:35:38.020419 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0410 16:35:38.020419 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0410 16:35:38.020428 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0410 16:35:38.020424 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0410 16:35:38.020443 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0410 16:35:38.020445 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0410 16:35:38.020448 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0410 16:35:38.020467 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0410 16:35:38.020472 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0410 16:35:38.020475 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0410 16:35:38.020480 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0410 16:35:38.020486 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0410 16:35:38.020488 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0410 16:35:38.020495 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0410 16:35:38.020505 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0410 16:35:38.020505 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0410 16:35:38.020504 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0410 16:35:38.020511 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0410 16:35:38.020530 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0410 16:35:38.020527 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0410 16:35:38.020533 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0410 16:35:38.020539 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0410 16:35:38.020546 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0410 16:35:38.020554 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0410 16:35:38.020557 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0410 16:35:38.020559 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0410 16:35:38.020530 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0410 16:35:38.020557 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0410 16:35:38.020568 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0410 16:35:38.020565 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0410 16:35:38.020578 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0410 16:35:38.020581 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0410 16:35:38.020584 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0410 16:35:38.020588 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0410 16:35:38.020590 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0410 16:35:38.020578 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0410 16:35:38.020599 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0410 16:35:38.020599 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0410 16:35:38.020602 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0410 16:35:38.020606 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0410 16:35:38.020612 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0410 16:35:38.020541 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0410 16:35:38.020618 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0410 16:35:38.020621 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0410 16:35:38.020625 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0410 16:35:38.020628 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0410 16:35:38.020631 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0410 16:35:38.020555 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0410 16:35:38.020641 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0410 16:35:38.020638 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0410 16:35:38.020587 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0410 16:35:38.020655 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0410 16:35:38.020663 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0410 16:35:38.020662 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0410 16:35:38.020559 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0410 16:35:38.020673 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0410 16:35:38.020668 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0410 16:35:38.020676 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0410 16:35:38.020680 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0410 16:35:38.020677 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0410 16:35:38.020686 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0410 16:35:38.020690 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0410 16:35:38.020697 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0410 16:35:38.020561 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0410 16:35:38.020699 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0410 16:35:38.020730 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0410 16:35:38.020680 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0410 16:35:38.020745 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0410 16:35:38.020667 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0410 16:35:38.020779 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0410 16:35:38.020790 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0410 16:35:38.020566 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0410 16:35:38.020647 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0410 16:35:38.020814 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0410 16:35:38.020575 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0410 16:35:38.020849 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0410 16:35:38.020848 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0410 16:35:38.020865 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0410 16:35:38.020871 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0410 16:35:38.020591 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0410 16:35:38.020560 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0410 16:35:38.020895 3933 port_claim.go:71] Svc openshift-ingress/router-default: port 31521 is already open, no action needed I0410 16:35:38.020611 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0410 16:35:38.020914 3933 port_claim.go:71] Svc openshift-ingress/router-default: port 31262 is already open, no action needed I0410 16:35:38.020634 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0410 16:35:38.020471 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0410 16:35:38.020952 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0410 16:35:38.020542 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0410 16:35:38.020957 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0410 16:35:38.020829 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0410 16:35:38.020969 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0410 16:35:38.020676 3933 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0410 16:35:38.021003 3933 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0410 16:35:38.063271 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0410 16:35:38.063290 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0410 16:35:38.063297 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0410 16:35:38.063304 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0410 16:35:38.063313 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0410 16:35:38.084278 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0410 16:35:38.084291 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0410 16:35:38.084309 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0410 16:35:38.084315 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0410 16:35:38.084322 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0410 16:35:38.084331 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0410 16:35:38.084337 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0410 16:35:38.084343 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0410 16:35:38.084348 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0410 16:35:38.084353 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0410 16:35:38.084358 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0410 16:35:38.084363 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0410 16:35:38.084368 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0410 16:35:38.126385 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0410 16:35:38.126394 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0410 16:35:38.126411 3933 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0410 16:35:38.518684 3933 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-52hhc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:35:38.518735 3933 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-52hhc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:35:38.556263 3933 default_network_controller.go:1008] Node "ip-10-0-1-177.ec2.internal" in remote zone "ip-10-0-1-177.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:38.556301 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-177.ec2.internal in network "default" I0410 16:35:38.556375 3933 node_tracker.go:235] Node ip-10-0-1-177.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-177.ec2.internal" I0410 16:35:38.556396 3933 node_tracker.go:169] Node ip-10-0-1-177.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:38.556410 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:38.582221 3933 default_network_controller.go:1008] Node "ip-10-0-1-177.ec2.internal" in remote zone "ip-10-0-1-177.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:38.729452 3933 default_network_controller.go:1008] Node "ip-10-0-1-177.ec2.internal" in remote zone "ip-10-0-1-177.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:38.729449 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-177.ec2.internal in network "default" I0410 16:35:38.729541 3933 node_tracker.go:169] Node ip-10-0-1-177.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:38.729558 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:38.730278 3933 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-177.ec2.internal for the network default I0410 16:35:38.733778 3933 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-177.ec2.internal" on network "default" took: 3.522741ms I0410 16:35:39.516144 3933 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-52hhc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:35:39.516222 3933 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-52hhc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:35:39.573054 3933 default_network_controller.go:1008] Node "ip-10-0-0-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:40.187276 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-187.ec2.internal I0410 16:35:40.187308 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-187.ec2.internal: 40.721µs I0410 16:35:40.213882 3933 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6667b64b75-tzs9k for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.214045 3933 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k I0410 16:35:40.222215 3933 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-76494886f6-k62gt for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.222312 3933 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt I0410 16:35:40.222401 3933 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz] creating logical port openshift-network-console_networking-console-plugin-d6f47c5fd-bfkzz for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.222476 3933 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-54b84b498b-tzqb9 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.222519 3933 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz I0410 16:35:40.222574 3933 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9 I0410 16:35:40.227033 3933 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-75b46cc85-zz6lg] creating logical port openshift-console-operator_console-operator-75b46cc85-zz6lg for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.227144 3933 kube.go:256] Updating pod openshift-console-operator/console-operator-75b46cc85-zz6lg I0410 16:35:40.227222 3933 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4] creating logical port openshift-monitoring_cluster-monitoring-operator-76dc995b8d-67ds4 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.227283 3933 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5bd7bdf744-866ds] creating logical port openshift-ingress_router-default-5bd7bdf744-866ds for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.227317 3933 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4 I0410 16:35:40.227424 3933 kube.go:256] Updating pod openshift-ingress/router-default-5bd7bdf744-866ds I0410 16:35:40.227834 3933 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl] creating logical port openshift-network-console_networking-console-plugin-d6f47c5fd-jzzsl for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.227929 3933 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl I0410 16:35:40.227985 3933 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-644bcb95-sxgb7] creating logical port openshift-ingress_router-default-644bcb95-sxgb7 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.228094 3933 kube.go:256] Updating pod openshift-ingress/router-default-644bcb95-sxgb7 I0410 16:35:40.246385 3933 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt] pod update took 24.103031ms I0410 16:35:40.246404 3933 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt] addLogicalPort annotation time took 24.125283ms I0410 16:35:40.246425 3933 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k] pod update took 32.417818ms I0410 16:35:40.246442 3933 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k] addLogicalPort annotation time took 32.439321ms I0410 16:35:40.247529 3933 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt] addLogicalPort took 25.330933ms, libovsdb time 897.204µs I0410 16:35:40.247899 3933 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k] addLogicalPort took 34.033556ms, libovsdb time 1.226439ms I0410 16:35:40.252777 3933 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9] pod update took 30.228774ms I0410 16:35:40.252798 3933 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9] addLogicalPort annotation time took 30.251017ms I0410 16:35:40.253934 3933 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9] addLogicalPort took 31.472268ms, libovsdb time 837.64µs I0410 16:35:40.254408 3933 pod.go:62] [openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz] pod update took 31.918659ms I0410 16:35:40.254424 3933 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz] addLogicalPort annotation time took 31.936392ms I0410 16:35:40.254707 3933 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7] creating logical port openshift-service-ca-operator_service-ca-operator-687fd84749-8z7z7 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.254822 3933 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7 I0410 16:35:40.255545 3933 pods.go:271] [openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz] addLogicalPort took 33.158474ms, libovsdb time 842.056µs I0410 16:35:40.255589 3933 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-595646bb87-8wq6f] creating logical port openshift-image-registry_image-registry-595646bb87-8wq6f for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.255666 3933 kube.go:256] Updating pod openshift-image-registry/image-registry-595646bb87-8wq6f I0410 16:35:40.255794 3933 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-7b4545dddc-8xgth] creating logical port openshift-insights_insights-operator-7b4545dddc-8xgth for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.255859 3933 kube.go:256] Updating pod openshift-insights/insights-operator-7b4545dddc-8xgth I0410 16:35:40.270561 3933 pod.go:62] [openshift-console-operator/console-operator-75b46cc85-zz6lg] pod update took 43.442831ms I0410 16:35:40.270587 3933 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-75b46cc85-zz6lg] addLogicalPort annotation time took 43.471931ms I0410 16:35:40.271599 3933 pods.go:271] [openshift-console-operator/console-operator-75b46cc85-zz6lg] addLogicalPort took 44.576138ms, libovsdb time 730.773µs I0410 16:35:40.280181 3933 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4] pod update took 52.884598ms I0410 16:35:40.280198 3933 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4] addLogicalPort annotation time took 52.905077ms I0410 16:35:40.280228 3933 pod.go:62] [openshift-ingress/router-default-644bcb95-sxgb7] pod update took 52.164602ms I0410 16:35:40.280250 3933 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-644bcb95-sxgb7] addLogicalPort annotation time took 52.188024ms I0410 16:35:40.281204 3933 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4] addLogicalPort took 53.989108ms, libovsdb time 776.261µs I0410 16:35:40.281377 3933 pods.go:271] [openshift-ingress/router-default-644bcb95-sxgb7] addLogicalPort took 53.400716ms, libovsdb time 881.25µs I0410 16:35:40.287817 3933 pod.go:62] [openshift-ingress/router-default-5bd7bdf744-866ds] pod update took 60.427478ms I0410 16:35:40.287835 3933 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5bd7bdf744-866ds] addLogicalPort annotation time took 60.444966ms I0410 16:35:40.287882 3933 pod.go:62] [openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl] pod update took 59.976116ms I0410 16:35:40.287902 3933 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl] addLogicalPort annotation time took 59.996519ms I0410 16:35:40.288885 3933 pods.go:271] [openshift-ingress/router-default-5bd7bdf744-866ds] addLogicalPort took 61.614385ms, libovsdb time 781.061µs I0410 16:35:40.289227 3933 pods.go:271] [openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl] addLogicalPort took 61.404778ms, libovsdb time 993.794µs I0410 16:35:40.295006 3933 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-sfpns] creating logical port openshift-dns_dns-default-sfpns for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.295014 3933 pod.go:62] [openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7] pod update took 40.208044ms I0410 16:35:40.295030 3933 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7] addLogicalPort annotation time took 40.224781ms I0410 16:35:40.295088 3933 kube.go:256] Updating pod openshift-dns/dns-default-sfpns I0410 16:35:40.296031 3933 pods.go:271] [openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7] addLogicalPort took 41.334637ms, libovsdb time 708.567µs I0410 16:35:40.297089 3933 pod.go:62] [openshift-insights/insights-operator-7b4545dddc-8xgth] pod update took 41.250542ms I0410 16:35:40.297104 3933 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-7b4545dddc-8xgth] addLogicalPort annotation time took 41.26737ms I0410 16:35:40.298090 3933 pods.go:271] [openshift-insights/insights-operator-7b4545dddc-8xgth] addLogicalPort took 42.304895ms, libovsdb time 690.269µs I0410 16:35:40.300263 3933 pod.go:62] [openshift-image-registry/image-registry-595646bb87-8wq6f] pod update took 44.623574ms I0410 16:35:40.300285 3933 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-595646bb87-8wq6f] addLogicalPort annotation time took 44.646705ms I0410 16:35:40.301337 3933 pods.go:271] [openshift-image-registry/image-registry-595646bb87-8wq6f] addLogicalPort took 45.754905ms, libovsdb time 716.599µs I0410 16:35:40.313673 3933 pod.go:62] [openshift-dns/dns-default-sfpns] pod update took 18.609433ms I0410 16:35:40.313688 3933 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-sfpns] addLogicalPort annotation time took 18.629903ms I0410 16:35:40.314737 3933 pods.go:271] [openshift-dns/dns-default-sfpns] addLogicalPort took 19.739735ms, libovsdb time 757.792µs I0410 16:35:40.316444 3933 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt] creating logical port openshift-network-diagnostics_network-check-source-57d556cf6f-mfwlt for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.316538 3933 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt I0410 16:35:40.318244 3933 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-jrjg7] creating logical port openshift-ingress-canary_ingress-canary-jrjg7 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.318324 3933 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-jrjg7 I0410 16:35:40.318546 3933 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-75zcr for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:40.318613 3933 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr I0410 16:35:40.333014 3933 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr] pod update took 14.41359ms I0410 16:35:40.333035 3933 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr] addLogicalPort annotation time took 14.437227ms I0410 16:35:40.333055 3933 pod.go:62] [openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt] pod update took 16.53708ms I0410 16:35:40.333078 3933 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt] addLogicalPort annotation time took 16.562735ms I0410 16:35:40.334196 3933 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr] addLogicalPort took 15.660629ms, libovsdb time 821.67µs I0410 16:35:40.334383 3933 pods.go:271] [openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt] addLogicalPort took 17.946473ms, libovsdb time 906.086µs I0410 16:35:40.336301 3933 pod.go:62] [openshift-ingress-canary/ingress-canary-jrjg7] pod update took 18.001345ms I0410 16:35:40.336322 3933 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-jrjg7] addLogicalPort annotation time took 18.024091ms I0410 16:35:40.337444 3933 pods.go:271] [openshift-ingress-canary/ingress-canary-jrjg7] addLogicalPort took 19.208522ms, libovsdb time 805.688µs I0410 16:35:40.622931 3933 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt e6bba039acd54f2b4965ec736cf334ac973765387848607fc0ccf99efbd93208 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt e6bba039acd54f2b4965ec736cf334ac973765387848607fc0ccf99efbd93208 network default NAD default] I0410 16:35:40.641048 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-76494886f6-k62gt, hostIfaceName: e6bba039acd54f2, network: default, NAD default, SandboxID: "e6bba039acd54f2b4965ec736cf334ac973765387848607fc0ccf99efbd93208", PCI device ID: , UID: "eef4bec7-cecf-4e79-8787-2ceb3d5c8c0f", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0410 16:35:40.646656 3933 cni.go:403] [openshift-console-operator/console-operator-75b46cc85-zz6lg 1719e8ea443abc49f21ea3b39d475cb646e7b4ad1967f48d335d1a2f223e1b1d network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-75b46cc85-zz6lg 1719e8ea443abc49f21ea3b39d475cb646e7b4ad1967f48d335d1a2f223e1b1d network default NAD default] I0410 16:35:40.656874 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-75b46cc85-zz6lg, hostIfaceName: 1719e8ea443abc4, network: default, NAD default, SandboxID: "1719e8ea443abc49f21ea3b39d475cb646e7b4ad1967f48d335d1a2f223e1b1d", PCI device ID: , UID: "128c6580-2371-49a5-896b-4a5500ad5620", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0410 16:35:40.692076 3933 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt e6bba039acd54f2b4965ec736cf334ac973765387848607fc0ccf99efbd93208 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-76494886f6-k62gt e6bba039acd54f2b4965ec736cf334ac973765387848607fc0ccf99efbd93208 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:d5:4b:02:30:0f\",\"name\":\"e6bba039acd54f2\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/024db595-a697-43b3-8423-df1fc2ae2b34\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.702186 3933 cni.go:424] [openshift-console-operator/console-operator-75b46cc85-zz6lg 1719e8ea443abc49f21ea3b39d475cb646e7b4ad1967f48d335d1a2f223e1b1d network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-75b46cc85-zz6lg 1719e8ea443abc49f21ea3b39d475cb646e7b4ad1967f48d335d1a2f223e1b1d network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:77:d0:d2:7a:81\",\"name\":\"1719e8ea443abc4\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc227640-20cb-4c18-8414-601e620e99dc\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.819607 3933 cni.go:403] [openshift-ingress-canary/ingress-canary-jrjg7 1d5110ba137de98bbc730982b790289a9ab6590098754d4d143e39a90c0594ec network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-jrjg7 1d5110ba137de98bbc730982b790289a9ab6590098754d4d143e39a90c0594ec network default NAD default] I0410 16:35:40.819653 3933 cni.go:403] [openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7 b880371b555dd961b9f72137352c66c95641216f61839fea12461566a8192e1f network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7 b880371b555dd961b9f72137352c66c95641216f61839fea12461566a8192e1f network default NAD default] I0410 16:35:40.819831 3933 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr 5dad0fc8a9cbf42e654b21c6eda4963445846ee12956103f0607b087a4ca6100 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr 5dad0fc8a9cbf42e654b21c6eda4963445846ee12956103f0607b087a4ca6100 network default NAD default] I0410 16:35:40.819999 3933 cni.go:403] [openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt a401703138277831d1ef08d8faec2aedf1561dc5eb958d005c22af3f391e3fa5 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt a401703138277831d1ef08d8faec2aedf1561dc5eb958d005c22af3f391e3fa5 network default NAD default] I0410 16:35:40.823277 3933 cni.go:403] [openshift-insights/insights-operator-7b4545dddc-8xgth 01e6767a94a29a81d22395104d874b1407a6d5ae2779c1c5355cb608cbdd465f network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-7b4545dddc-8xgth 01e6767a94a29a81d22395104d874b1407a6d5ae2779c1c5355cb608cbdd465f network default NAD default] I0410 16:35:40.840765 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-687fd84749-8z7z7, hostIfaceName: b880371b555dd96, network: default, NAD default, SandboxID: "b880371b555dd961b9f72137352c66c95641216f61839fea12461566a8192e1f", PCI device ID: , UID: "4f501ee1-4376-43fe-b513-240198078632", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0410 16:35:40.841901 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-57d556cf6f-mfwlt, hostIfaceName: a40170313827783, network: default, NAD default, SandboxID: "a401703138277831d1ef08d8faec2aedf1561dc5eb958d005c22af3f391e3fa5", PCI device ID: , UID: "8cc048f8-1aaa-419e-87cd-77153c5e3849", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0410 16:35:40.842343 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-7b4545dddc-8xgth, hostIfaceName: 01e6767a94a29a8, network: default, NAD default, SandboxID: "01e6767a94a29a81d22395104d874b1407a6d5ae2779c1c5355cb608cbdd465f", PCI device ID: , UID: "93195b24-9452-4768-b8a4-a3e7b5720315", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0410 16:35:40.843963 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-jrjg7, hostIfaceName: 1d5110ba137de98, network: default, NAD default, SandboxID: "1d5110ba137de98bbc730982b790289a9ab6590098754d4d143e39a90c0594ec", PCI device ID: , UID: "3c27988f-7e3d-40a7-8c32-91a2de079dc3", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0410 16:35:40.853062 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-75zcr, hostIfaceName: 5dad0fc8a9cbf42, network: default, NAD default, SandboxID: "5dad0fc8a9cbf42e654b21c6eda4963445846ee12956103f0607b087a4ca6100", PCI device ID: , UID: "9ba07861-c4b7-4229-b85f-ea74a8d8551b", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0410 16:35:40.903665 3933 cni.go:424] [openshift-insights/insights-operator-7b4545dddc-8xgth 01e6767a94a29a81d22395104d874b1407a6d5ae2779c1c5355cb608cbdd465f network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-7b4545dddc-8xgth 01e6767a94a29a81d22395104d874b1407a6d5ae2779c1c5355cb608cbdd465f network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:6a:eb:12:1e:45\",\"name\":\"01e6767a94a29a8\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/411adb07-7337-42fe-a25f-fe0d4a8967a5\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.909709 3933 cni.go:424] [openshift-ingress-canary/ingress-canary-jrjg7 1d5110ba137de98bbc730982b790289a9ab6590098754d4d143e39a90c0594ec network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-jrjg7 1d5110ba137de98bbc730982b790289a9ab6590098754d4d143e39a90c0594ec network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:f1:ca:82:a0:45\",\"name\":\"1d5110ba137de98\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad94aa9a-17d6-4967-b369-818f63d389a2\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.910785 3933 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k 1c3c8a2c5c5157f1b59e61b6bf891189798c84e6259af4b60f499fee8f0e872b network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k 1c3c8a2c5c5157f1b59e61b6bf891189798c84e6259af4b60f499fee8f0e872b network default NAD default] I0410 16:35:40.915193 3933 cni.go:424] [openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt a401703138277831d1ef08d8faec2aedf1561dc5eb958d005c22af3f391e3fa5 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-57d556cf6f-mfwlt a401703138277831d1ef08d8faec2aedf1561dc5eb958d005c22af3f391e3fa5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:c1:8f:6e:b4:c0\",\"name\":\"a40170313827783\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6cdafa43-0071-4def-85a8-33cee723f8a7\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.915372 3933 cni.go:424] [openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7 b880371b555dd961b9f72137352c66c95641216f61839fea12461566a8192e1f network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-687fd84749-8z7z7 b880371b555dd961b9f72137352c66c95641216f61839fea12461566a8192e1f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:99:3b:d0:6e:ba\",\"name\":\"b880371b555dd96\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2499e85-53bb-46c5-98c6-0cd0f8fd3b4c\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.921196 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6667b64b75-tzs9k, hostIfaceName: 1c3c8a2c5c5157f, network: default, NAD default, SandboxID: "1c3c8a2c5c5157f1b59e61b6bf891189798c84e6259af4b60f499fee8f0e872b", PCI device ID: , UID: "d6584fae-85fa-4da6-82bb-6daaa4294251", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0410 16:35:40.926095 3933 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr 5dad0fc8a9cbf42e654b21c6eda4963445846ee12956103f0607b087a4ca6100 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75zcr 5dad0fc8a9cbf42e654b21c6eda4963445846ee12956103f0607b087a4ca6100 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:37:44:22:bf:22\",\"name\":\"5dad0fc8a9cbf42\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39866f0d-ba95-49b7-aafe-714bd8bba1aa\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:40.972257 3933 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k 1c3c8a2c5c5157f1b59e61b6bf891189798c84e6259af4b60f499fee8f0e872b network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6667b64b75-tzs9k 1c3c8a2c5c5157f1b59e61b6bf891189798c84e6259af4b60f499fee8f0e872b network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:7e:66:d4:c8:60\",\"name\":\"1c3c8a2c5c5157f\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8719c401-c0fa-4f22-af0e-ce3395a60159\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:41.288924 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-177.ec2.internal in network "default" I0410 16:35:41.288939 3933 egressfirewall.go:628] Syncing node "ip-10-0-1-177.ec2.internal" for egress firewall I0410 16:35:41.288988 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-177.ec2.internal I0410 16:35:41.288996 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-177.ec2.internal", IP: 10.0.1.177 to PMTUD blocking rules I0410 16:35:41.289003 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-177.ec2.internal: 21.151µs I0410 16:35:41.289009 3933 node_tracker.go:169] Node ip-10-0-1-177.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:41.289010 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-177.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0410 16:35:41.289024 3933 services_controller.go:574] Full service sync requested for network=default I0410 16:35:41.295374 3933 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-177.ec2.internal. Updating no re-route policies I0410 16:35:41.305722 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-177.ec2.internal", IP: 10.0.1.177 to PMTUD blocking rules I0410 16:35:41.305742 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-1-177.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0410 16:35:45.752950 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-177.ec2.internal I0410 16:35:45.752989 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-177.ec2.internal: 44.484µs I0410 16:35:46.003456 3933 cni.go:403] [openshift-network-diagnostics/network-check-target-ch6vc 905e606dd0fc95615f97031a4a6289d01e138ccf5084031b7c6ee5719b036f06 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-ch6vc 905e606dd0fc95615f97031a4a6289d01e138ccf5084031b7c6ee5719b036f06 network default NAD default] I0410 16:35:46.014662 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-ch6vc, hostIfaceName: 905e606dd0fc956, network: default, NAD default, SandboxID: "905e606dd0fc95615f97031a4a6289d01e138ccf5084031b7c6ee5719b036f06", PCI device ID: , UID: "703ea8fd-1554-4c29-8969-c8e83911e927", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0410 16:35:46.070526 3933 cni.go:424] [openshift-network-diagnostics/network-check-target-ch6vc 905e606dd0fc95615f97031a4a6289d01e138ccf5084031b7c6ee5719b036f06 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-ch6vc 905e606dd0fc95615f97031a4a6289d01e138ccf5084031b7c6ee5719b036f06 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:1a:62:74:d4:cf\",\"name\":\"905e606dd0fc956\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e54df6c6-aed0-46c5-b6df-b64a7d5ea29d\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0410 16:35:46.694997 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0410 16:35:47.634826 3933 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0410 16:35:47.661933 3933 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0410 16:35:47.693663 3933 cni.go:403] [kube-system/global-pull-secret-syncer-8sxml b17ffc6a28b422b146ed87e4fe9c2276179cc0c77dc7ff9b46c08e63506e87f7 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-8sxml b17ffc6a28b422b146ed87e4fe9c2276179cc0c77dc7ff9b46c08e63506e87f7 network default NAD default] I0410 16:35:47.705866 3933 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-8sxml, hostIfaceName: b17ffc6a28b422b, network: default, NAD default, SandboxID: "b17ffc6a28b422b146ed87e4fe9c2276179cc0c77dc7ff9b46c08e63506e87f7", PCI device ID: , UID: "86d7f0c4-4ea5-4a82-ae00-791f50ab70db", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0410 16:35:47.750563 3933 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0410 16:35:47.751351 3933 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 768.023µs I0410 16:35:47.755026 3933 cni.go:424] [kube-system/global-pull-secret-syncer-8sxml b17ffc6a28b422b146ed87e4fe9c2276179cc0c77dc7ff9b46c08e63506e87f7 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-8sxml b17ffc6a28b422b146ed87e4fe9c2276179cc0c77dc7ff9b46c08e63506e87f7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:22:1a:b8:d2:20\",\"name\":\"b17ffc6a28b422b\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f66450d6-d0ed-4de9-839c-dcc5bc8efe87\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:35:47.770236 3933 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0410 16:35:48.254957 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-66.ec2.internal I0410 16:35:48.254983 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-66.ec2.internal: 33.67µs I0410 16:35:48.255010 3933 egressfirewall.go:628] Syncing node "ip-10-0-0-66.ec2.internal" for egress firewall I0410 16:35:48.255025 3933 default_network_controller.go:1008] Node "ip-10-0-0-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:48.799269 3933 namespace.go:98] [openshift-service-ca] adding namespace I0410 16:35:48.800259 3933 namespace.go:102] [openshift-service-ca] adding namespace took 966.695µs I0410 16:35:48.815977 3933 namespace.go:142] [openshift-service-ca] updating namespace W0410 16:35:52.632251 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0410 16:35:53.500033 3933 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0410 16:35:53.601689 3933 default_network_controller.go:1008] Node "ip-10-0-0-66.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:53.627957 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-66.ec2.internal in network "default" I0410 16:35:53.628008 3933 node_tracker.go:235] Node ip-10-0-0-66.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-66.ec2.internal" I0410 16:35:53.628031 3933 node_tracker.go:169] Node ip-10-0-0-66.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:53.627956 3933 default_network_controller.go:1008] Node "ip-10-0-0-66.ec2.internal" in remote zone "ip-10-0-0-66.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:53.628052 3933 services_controller.go:574] Full service sync requested for network=default W0410 16:35:53.628144 3933 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0410 16:35:53.628382 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0410 16:35:53.780474 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-66.ec2.internal in network "default" I0410 16:35:53.780508 3933 default_network_controller.go:1008] Node "ip-10-0-0-66.ec2.internal" in remote zone "ip-10-0-0-66.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0410 16:35:53.780536 3933 node_tracker.go:169] Node ip-10-0-0-66.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:53.780553 3933 services_controller.go:574] Full service sync requested for network=default W0410 16:35:53.780646 3933 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0410 16:35:53.780887 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0410 16:35:53.781247 3933 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-66.ec2.internal for the network default I0410 16:35:53.782557 3933 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-66.ec2.internal" on network "default" took: 1.323752ms I0410 16:35:55.337201 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-66.ec2.internal I0410 16:35:55.337226 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-66.ec2.internal: 31.59µs I0410 16:35:55.337289 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-66.ec2.internal", IP: 10.0.0.66 to PMTUD blocking rules I0410 16:35:55.337303 3933 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-66.ec2.internal in network "default" I0410 16:35:55.337313 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-66.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0410 16:35:55.337324 3933 egressfirewall.go:628] Syncing node "ip-10-0-0-66.ec2.internal" for egress firewall I0410 16:35:55.337363 3933 node_tracker.go:169] Node ip-10-0-0-66.ec2.internal switch + router changed, syncing services in network "default" I0410 16:35:55.337388 3933 services_controller.go:574] Full service sync requested for network=default W0410 16:35:55.337544 3933 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0410 16:35:55.337838 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0410 16:35:55.344516 3933 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-66.ec2.internal. Updating no re-route policies I0410 16:35:55.352219 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-66.ec2.internal", IP: 10.0.0.66 to PMTUD blocking rules I0410 16:35:55.352236 3933 default_node_network_controller.go:1282] Adding remote node "ip-10-0-0-66.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0410 16:35:55.993016 3933 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-hv264] creating logical port openshift-insights_insights-runtime-extractor-hv264 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:35:55.993147 3933 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-hv264 I0410 16:35:56.007305 3933 pod.go:62] [openshift-insights/insights-runtime-extractor-hv264] pod update took 14.198788ms I0410 16:35:56.007326 3933 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-hv264] addLogicalPort annotation time took 14.226407ms I0410 16:35:56.008624 3933 pods.go:271] [openshift-insights/insights-runtime-extractor-hv264] addLogicalPort took 15.624992ms, libovsdb time 935.549µs I0410 16:35:59.134560 3933 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-66.ec2.internal I0410 16:35:59.134589 3933 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-66.ec2.internal: 35.615µs W0410 16:35:59.441130 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0410 16:36:03.892000 3933 cni.go:403] [openshift-insights/insights-runtime-extractor-hv264 edeed153621b59e3bc7f2eb7888181e57d45e67967d09ce68197c04c3cba0bb5 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-hv264 edeed153621b59e3bc7f2eb7888181e57d45e67967d09ce68197c04c3cba0bb5 network default NAD default] I0410 16:36:03.900949 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-hv264, hostIfaceName: edeed153621b59e, network: default, NAD default, SandboxID: "edeed153621b59e3bc7f2eb7888181e57d45e67967d09ce68197c04c3cba0bb5", PCI device ID: , UID: "db89c1c9-cb2c-4276-9fc1-8b6dd725109e", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0410 16:36:03.949857 3933 cni.go:424] [openshift-insights/insights-runtime-extractor-hv264 edeed153621b59e3bc7f2eb7888181e57d45e67967d09ce68197c04c3cba0bb5 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-hv264 edeed153621b59e3bc7f2eb7888181e57d45e67967d09ce68197c04c3cba0bb5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:dd:6c:43:56:37\",\"name\":\"edeed153621b59e\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/054273ee-a1fb-4638-b514-0fda79fce12e\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.404185 3933 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9 2e988009971cd0243dec97abd9c85f54b0f8339c9ad9c7e01b2ef0ae5db9eee3 network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9 2e988009971cd0243dec97abd9c85f54b0f8339c9ad9c7e01b2ef0ae5db9eee3 network default NAD default] I0410 16:36:12.404591 3933 cni.go:403] [openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz 2c341416c2fe1869687e25fc72992ce518e890419c9d0b7db4dfe6a730d8d247 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz 2c341416c2fe1869687e25fc72992ce518e890419c9d0b7db4dfe6a730d8d247 network default NAD default] I0410 16:36:12.415139 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-d6f47c5fd-bfkzz, hostIfaceName: 2c341416c2fe186, network: default, NAD default, SandboxID: "2c341416c2fe1869687e25fc72992ce518e890419c9d0b7db4dfe6a730d8d247", PCI device ID: , UID: "513d4236-0318-4f97-8a05-b24c26026934", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0410 16:36:12.415618 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-54b84b498b-tzqb9, hostIfaceName: 2e988009971cd02, network: default, NAD default, SandboxID: "2e988009971cd0243dec97abd9c85f54b0f8339c9ad9c7e01b2ef0ae5db9eee3", PCI device ID: , UID: "4d841779-1ce9-4f72-82b2-7f59d5b12c51", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0410 16:36:12.427250 3933 cni.go:403] [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default] I0410 16:36:12.439534 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-595646bb87-8wq6f, hostIfaceName: 3155ce920e44b99, network: default, NAD default, SandboxID: "3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6", PCI device ID: , UID: "e271e8e9-18af-4596-9d7c-75f11c054c18", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0410 16:36:12.474647 3933 cni.go:424] [openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz 2c341416c2fe1869687e25fc72992ce518e890419c9d0b7db4dfe6a730d8d247 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-d6f47c5fd-bfkzz 2c341416c2fe1869687e25fc72992ce518e890419c9d0b7db4dfe6a730d8d247 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:fb:af:aa:a7:5a\",\"name\":\"2c341416c2fe186\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6fdebff3-7bd8-4de9-83c6-77163a424a3a\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.475290 3933 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4 7171d18bd854c06894a2d007fd1fe41cc78f554d406d074443698ee38dd570ab network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4 7171d18bd854c06894a2d007fd1fe41cc78f554d406d074443698ee38dd570ab network default NAD default] I0410 16:36:12.487990 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-76dc995b8d-67ds4, hostIfaceName: 7171d18bd854c06, network: default, NAD default, SandboxID: "7171d18bd854c06894a2d007fd1fe41cc78f554d406d074443698ee38dd570ab", PCI device ID: , UID: "23b8f7f2-094d-455f-a951-51cdb5ff0d94", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0410 16:36:12.492073 3933 cni.go:403] [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default] ADD starting CNI request [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default] I0410 16:36:12.495400 3933 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9 2e988009971cd0243dec97abd9c85f54b0f8339c9ad9c7e01b2ef0ae5db9eee3 network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-54b84b498b-tzqb9 2e988009971cd0243dec97abd9c85f54b0f8339c9ad9c7e01b2ef0ae5db9eee3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:1f:29:02:86:1a\",\"name\":\"2e988009971cd02\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13ed0ef1-755b-4ef9-98f8-4b2e9b7e65ab\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.502321 3933 cni.go:403] [openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl f47593fbec4f939bccf191003b39bf7f4ff2868fba97427d54e4ecae78e5ee6b network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl f47593fbec4f939bccf191003b39bf7f4ff2868fba97427d54e4ecae78e5ee6b network default NAD default] I0410 16:36:12.509771 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5bd7bdf744-866ds, hostIfaceName: 418ec18095addfc, network: default, NAD default, SandboxID: "418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de", PCI device ID: , UID: "1aa4fbb0-2487-44e3-9331-f9a94f2e9f91", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0410 16:36:12.514897 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-d6f47c5fd-jzzsl, hostIfaceName: f47593fbec4f939, network: default, NAD default, SandboxID: "f47593fbec4f939bccf191003b39bf7f4ff2868fba97427d54e4ecae78e5ee6b", PCI device ID: , UID: "dcd707a7-c069-4259-a5d9-f623f494155b", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0410 16:36:12.522072 3933 cni.go:424] [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:9a:e7:4a:f7:1c\",\"name\":\"3155ce920e44b99\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2cac8d28-80b0-45a6-841d-e0dd36aaf1b7\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.538720 3933 cni.go:403] [openshift-ingress/router-default-644bcb95-sxgb7 d42102b01823bdc7b7ac7b8f9f1d2869fefe8be5bc65a62afbb64f5e5d6c74f8 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-644bcb95-sxgb7 d42102b01823bdc7b7ac7b8f9f1d2869fefe8be5bc65a62afbb64f5e5d6c74f8 network default NAD default] I0410 16:36:12.549665 3933 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4 7171d18bd854c06894a2d007fd1fe41cc78f554d406d074443698ee38dd570ab network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-76dc995b8d-67ds4 7171d18bd854c06894a2d007fd1fe41cc78f554d406d074443698ee38dd570ab network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:93:58:c8:9c:69\",\"name\":\"7171d18bd854c06\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39b50b2e-f827-41f6-b899-32b9d7c51814\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.552199 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-644bcb95-sxgb7, hostIfaceName: d42102b01823bdc, network: default, NAD default, SandboxID: "d42102b01823bdc7b7ac7b8f9f1d2869fefe8be5bc65a62afbb64f5e5d6c74f8", PCI device ID: , UID: "30b02a3f-e229-443e-86d7-ab9eb8dc984f", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0410 16:36:12.571639 3933 cni.go:424] [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default] ADD finished CNI request [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:0d:d4:55:19:a9\",\"name\":\"418ec18095addfc\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6e9475be-c284-4860-8934-0295dd12ead1\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.572650 3933 cni.go:403] [openshift-dns/dns-default-sfpns 8322e8978638cfd61404a1e96405b366e663154d5869a455f5c39e5c8744af30 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-sfpns 8322e8978638cfd61404a1e96405b366e663154d5869a455f5c39e5c8744af30 network default NAD default] I0410 16:36:12.593315 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-sfpns, hostIfaceName: 8322e8978638cfd, network: default, NAD default, SandboxID: "8322e8978638cfd61404a1e96405b366e663154d5869a455f5c39e5c8744af30", PCI device ID: , UID: "f1b186e9-7581-417b-b85b-54746cd1c84b", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0410 16:36:12.603180 3933 cni.go:424] [openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl f47593fbec4f939bccf191003b39bf7f4ff2868fba97427d54e4ecae78e5ee6b network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-d6f47c5fd-jzzsl f47593fbec4f939bccf191003b39bf7f4ff2868fba97427d54e4ecae78e5ee6b network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:98:fa:d3:26:3b\",\"name\":\"f47593fbec4f939\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f316ed03-b3e6-4398-beeb-0f7ad2cf8c5a\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.621593 3933 cni.go:424] [openshift-ingress/router-default-644bcb95-sxgb7 d42102b01823bdc7b7ac7b8f9f1d2869fefe8be5bc65a62afbb64f5e5d6c74f8 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-644bcb95-sxgb7 d42102b01823bdc7b7ac7b8f9f1d2869fefe8be5bc65a62afbb64f5e5d6c74f8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:d5:a3:b9:a4:87\",\"name\":\"d42102b01823bdc\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22647c58-adb7-4a59-9ab6-3cd10457de76\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:12.651128 3933 cni.go:424] [openshift-dns/dns-default-sfpns 8322e8978638cfd61404a1e96405b366e663154d5869a455f5c39e5c8744af30 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-sfpns 8322e8978638cfd61404a1e96405b366e663154d5869a455f5c39e5c8744af30 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:a9:56:a4:85:57\",\"name\":\"8322e8978638cfd\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1690c71e-2710-4821-8155-bf3f5897dd32\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:16.346561 3933 cni.go:403] [openshift-multus/network-metrics-daemon-vfw65 6ab3bfafbee8a41578be9adb42476493136672a4e939998f5152eba25fa60f4c network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-vfw65 6ab3bfafbee8a41578be9adb42476493136672a4e939998f5152eba25fa60f4c network default NAD default] I0410 16:36:16.356209 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-vfw65, hostIfaceName: 6ab3bfafbee8a41, network: default, NAD default, SandboxID: "6ab3bfafbee8a41578be9adb42476493136672a4e939998f5152eba25fa60f4c", PCI device ID: , UID: "95d32221-a46f-4f10-bf70-46a71bec9927", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0410 16:36:16.410030 3933 cni.go:424] [openshift-multus/network-metrics-daemon-vfw65 6ab3bfafbee8a41578be9adb42476493136672a4e939998f5152eba25fa60f4c network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-vfw65 6ab3bfafbee8a41578be9adb42476493136672a4e939998f5152eba25fa60f4c network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f7:63:24:e4:9a\",\"name\":\"6ab3bfafbee8a41\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/88c726a5-99f1-41df-8a55-000bda3821ed\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0410 16:36:17.627811 3933 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0410 16:36:17.640033 3933 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0410 16:36:17.736941 3933 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0410 16:36:17.750105 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0410 16:36:17.750121 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0410 16:36:17.750127 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0410 16:36:17.786567 3933 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0410 16:36:18.594031 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0410 16:36:19.437197 3933 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b754c9d8d-wzkft of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:36:19.437814 3933 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b754c9d8d-wzkft of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:36:19.449005 3933 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b754c9d8d-wzkft of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:36:19.449027 3933 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b754c9d8d-wzkft of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:36:19.459200 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5b754c9d8d-wzkft I0410 16:36:19.459226 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5b754c9d8d-wzkft W0410 16:36:19.601477 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0410 16:36:22.481942 3933 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0410 16:36:23.549688 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0410 16:36:23.549714 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0410 16:36:26.742504 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0410 16:36:26.742528 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0410 16:36:26.742537 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0410 16:36:30.704232 3933 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0410 16:36:30.704254 3933 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0410 16:36:30.704263 3933 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0410 16:36:31.539886 3933 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0410 16:36:31.539912 3933 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0410 16:36:31.539919 3933 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0410 16:36:32.447953 3933 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-599546947f-7cpf6] creating logical port openshift-monitoring_monitoring-plugin-599546947f-7cpf6 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:36:32.448044 3933 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-599546947f-7cpf6 I0410 16:36:32.465969 3933 pod.go:62] [openshift-monitoring/monitoring-plugin-599546947f-7cpf6] pod update took 17.956868ms I0410 16:36:32.465990 3933 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-599546947f-7cpf6] addLogicalPort annotation time took 17.981745ms I0410 16:36:32.467129 3933 pods.go:271] [openshift-monitoring/monitoring-plugin-599546947f-7cpf6] addLogicalPort took 19.190903ms, libovsdb time 806.49µs I0410 16:36:32.808293 3933 cni.go:403] [openshift-monitoring/monitoring-plugin-599546947f-7cpf6 506ec48ab4f16301cb5bca05989a42c6fe970d2ea3c3acab7ad368791b3325a3 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-599546947f-7cpf6 506ec48ab4f16301cb5bca05989a42c6fe970d2ea3c3acab7ad368791b3325a3 network default NAD default] I0410 16:36:32.817639 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-599546947f-7cpf6, hostIfaceName: 506ec48ab4f1630, network: default, NAD default, SandboxID: "506ec48ab4f16301cb5bca05989a42c6fe970d2ea3c3acab7ad368791b3325a3", PCI device ID: , UID: "7a8788d2-307f-40c7-82f0-ffa56d05a657", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0410 16:36:32.866797 3933 cni.go:424] [openshift-monitoring/monitoring-plugin-599546947f-7cpf6 506ec48ab4f16301cb5bca05989a42c6fe970d2ea3c3acab7ad368791b3325a3 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-599546947f-7cpf6 506ec48ab4f16301cb5bca05989a42c6fe970d2ea3c3acab7ad368791b3325a3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:3b:9d:37:67:87\",\"name\":\"506ec48ab4f1630\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e22e3dec-1615-48c1-9a73-61d29331add0\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0410 16:36:34.681483 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0410 16:36:34.752215 3933 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0410 16:36:34.752233 3933 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0410 16:36:34.771028 3933 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:34.771046 3933 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:34.771053 3933 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:34.771059 3933 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:35.053791 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0410 16:36:35.221297 3933 base_network_controller_pods.go:487] [default/openshift-console/downloads-59f56c7685-xt2kg] creating logical port openshift-console_downloads-59f56c7685-xt2kg for pod on switch ip-10-0-2-187.ec2.internal I0410 16:36:35.221443 3933 kube.go:256] Updating pod openshift-console/downloads-59f56c7685-xt2kg I0410 16:36:35.239300 3933 pod.go:62] [openshift-console/downloads-59f56c7685-xt2kg] pod update took 17.88833ms I0410 16:36:35.239321 3933 base_network_controller_pods.go:937] [default/openshift-console/downloads-59f56c7685-xt2kg] addLogicalPort annotation time took 17.914954ms I0410 16:36:35.240479 3933 pods.go:271] [openshift-console/downloads-59f56c7685-xt2kg] addLogicalPort took 19.205043ms, libovsdb time 828.12µs W0410 16:36:35.556106 3933 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:35.556130 3933 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:35.556138 3933 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:35.556144 3933 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0410 16:36:35.575485 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0410 16:36:35.581795 3933 cni.go:403] [openshift-console/downloads-59f56c7685-xt2kg 385fbf7a79d7edc2a5d52cbd3aba602c548b49dc1aac62217d22a7fffa94adef network default NAD default] ADD starting CNI request [openshift-console/downloads-59f56c7685-xt2kg 385fbf7a79d7edc2a5d52cbd3aba602c548b49dc1aac62217d22a7fffa94adef network default NAD default] I0410 16:36:35.591623 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-59f56c7685-xt2kg, hostIfaceName: 385fbf7a79d7edc, network: default, NAD default, SandboxID: "385fbf7a79d7edc2a5d52cbd3aba602c548b49dc1aac62217d22a7fffa94adef", PCI device ID: , UID: "b655f5b0-b64b-4b13-8474-93ab2bcc86d2", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] W0410 16:36:35.608694 3933 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0410 16:36:35.608707 3933 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0410 16:36:35.639818 3933 cni.go:424] [openshift-console/downloads-59f56c7685-xt2kg 385fbf7a79d7edc2a5d52cbd3aba602c548b49dc1aac62217d22a7fffa94adef network default NAD default] ADD finished CNI request [openshift-console/downloads-59f56c7685-xt2kg 385fbf7a79d7edc2a5d52cbd3aba602c548b49dc1aac62217d22a7fffa94adef network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:69:be:51:e3:ad\",\"name\":\"385fbf7a79d7edc\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42ff9229-1384-4f65-9068-2bf64287aec2\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0410 16:36:35.843085 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin I0410 16:36:42.943110 3933 cni.go:403] [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default] I0410 16:36:43.019396 3933 cni.go:424] [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-595646bb87-8wq6f 3155ce920e44b99d7680a9598eb8456b08ab8c82c95e50003c7680d45cd0a6a6 network default NAD default], result "{}", err I0410 16:36:43.070277 3933 obj_retry.go:464] Detected object openshift-image-registry/image-registry-595646bb87-8wq6f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:36:43.070322 3933 pods.go:176] Deleting pod: openshift-image-registry/image-registry-595646bb87-8wq6f I0410 16:36:43.071801 3933 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-595646bb87-8wq6f, ips: 10.128.0.16 I0410 16:36:43.071838 3933 obj_retry.go:464] Detected object openshift-image-registry/image-registry-595646bb87-8wq6f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:36:43.081211 3933 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-56899bf957-tb7bh] creating logical port openshift-image-registry_image-registry-56899bf957-tb7bh for pod on switch ip-10-0-2-187.ec2.internal I0410 16:36:43.081296 3933 kube.go:256] Updating pod openshift-image-registry/image-registry-56899bf957-tb7bh I0410 16:36:43.097928 3933 pod.go:62] [openshift-image-registry/image-registry-56899bf957-tb7bh] pod update took 16.657286ms I0410 16:36:43.097957 3933 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-56899bf957-tb7bh] addLogicalPort annotation time took 16.689094ms I0410 16:36:43.099414 3933 pods.go:271] [openshift-image-registry/image-registry-56899bf957-tb7bh] addLogicalPort took 18.22996ms, libovsdb time 964.524µs I0410 16:36:43.521108 3933 base_network_controller_pods.go:487] [default/openshift-console/console-5f495d9d9c-tq25g] creating logical port openshift-console_console-5f495d9d9c-tq25g for pod on switch ip-10-0-2-187.ec2.internal I0410 16:36:43.521294 3933 kube.go:256] Updating pod openshift-console/console-5f495d9d9c-tq25g I0410 16:36:43.544729 3933 pod.go:62] [openshift-console/console-5f495d9d9c-tq25g] pod update took 23.471941ms I0410 16:36:43.544811 3933 base_network_controller_pods.go:937] [default/openshift-console/console-5f495d9d9c-tq25g] addLogicalPort annotation time took 23.557824ms I0410 16:36:43.546050 3933 pods.go:271] [openshift-console/console-5f495d9d9c-tq25g] addLogicalPort took 24.957055ms, libovsdb time 767.979µs I0410 16:36:43.854052 3933 obj_retry.go:458] Detected object openshift-image-registry/image-registry-595646bb87-8wq6f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:36:43.854076 3933 obj_retry.go:458] Detected object openshift-image-registry/image-registry-595646bb87-8wq6f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:36:43.862051 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-595646bb87-8wq6f I0410 16:36:43.862077 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-595646bb87-8wq6f I0410 16:36:44.140233 3933 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0410 16:36:50.021609 3933 cni.go:403] [openshift-image-registry/image-registry-56899bf957-tb7bh 40f6bf7679df5e4ff761de61a29f4aeb43e7cd6ac8374929c89fd510fdbf729b network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-56899bf957-tb7bh 40f6bf7679df5e4ff761de61a29f4aeb43e7cd6ac8374929c89fd510fdbf729b network default NAD default] I0410 16:36:50.024597 3933 cni.go:403] [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default] ADD starting CNI request [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default] I0410 16:36:50.031656 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-56899bf957-tb7bh, hostIfaceName: 40f6bf7679df5e4, network: default, NAD default, SandboxID: "40f6bf7679df5e4ff761de61a29f4aeb43e7cd6ac8374929c89fd510fdbf729b", PCI device ID: , UID: "962723a2-5cae-4224-99a7-ef944cfb9898", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0410 16:36:50.034165 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5f495d9d9c-tq25g, hostIfaceName: 73534e9c8b1fffa, network: default, NAD default, SandboxID: "73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24", PCI device ID: , UID: "0361899e-d5a7-4788-bb42-71b20436fa82", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0410 16:36:50.083766 3933 cni.go:424] [openshift-image-registry/image-registry-56899bf957-tb7bh 40f6bf7679df5e4ff761de61a29f4aeb43e7cd6ac8374929c89fd510fdbf729b network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-56899bf957-tb7bh 40f6bf7679df5e4ff761de61a29f4aeb43e7cd6ac8374929c89fd510fdbf729b network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:e5:61:07:4e:40\",\"name\":\"40f6bf7679df5e4\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9a9bcda-a9ed-4a02-88e0-05a9561549ba\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:36:50.107337 3933 cni.go:424] [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default] ADD finished CNI request [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:09:69:70:5e:91\",\"name\":\"73534e9c8b1fffa\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1d37ce98-63e7-420d-91c9-91872ed93170\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0410 16:36:50.884977 3933 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0410 16:36:50.942895 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0410 16:36:52.669197 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0410 16:36:52.899815 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0410 16:36:53.783921 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0410 16:36:53.915091 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0410 16:37:08.857517 3933 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b754c9d8d-s6jhz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:08.858058 3933 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5b754c9d8d-s6jhz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:09.836085 3933 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b754c9d8d-s6jhz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:09.836113 3933 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5b754c9d8d-s6jhz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:09.839757 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5b754c9d8d-s6jhz I0410 16:37:09.839787 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5b754c9d8d-s6jhz I0410 16:37:10.886188 3933 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:10.886736 3933 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:10.886766 3933 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0410 16:37:10.886782 3933 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0410 16:37:10.886799 3933 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0410 16:37:10.886807 3933 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0410 16:37:11.903961 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0410 16:37:11.915341 3933 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:11.915358 3933 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:11.915377 3933 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:11.915390 3933 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390dms4l of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0410 16:37:11.993026 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0410 16:37:11.993028 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0410 16:37:11.993075 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0410 16:37:13.007008 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0410 16:37:13.034998 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0410 16:37:13.035010 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0410 16:37:13.035054 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0410 16:37:14.002884 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0410 16:37:14.361739 3933 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0410 16:37:14.370699 3933 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0410 16:37:15.960577 3933 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-qwqc2 for pod on switch ip-10-0-2-187.ec2.internal I0410 16:37:15.960731 3933 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2 I0410 16:37:15.978092 3933 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2] pod update took 17.390631ms I0410 16:37:15.978117 3933 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2] addLogicalPort annotation time took 17.421982ms I0410 16:37:15.979453 3933 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2] addLogicalPort took 18.900369ms, libovsdb time 923.445µs I0410 16:37:16.372291 3933 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2 b745fb601db8fc8c0068c011db666d813d9ea14dcd84c6bc29b718cd2ff4af94 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2 b745fb601db8fc8c0068c011db666d813d9ea14dcd84c6bc29b718cd2ff4af94 network default NAD default] I0410 16:37:16.381658 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-qwqc2, hostIfaceName: b745fb601db8fc8, network: default, NAD default, SandboxID: "b745fb601db8fc8c0068c011db666d813d9ea14dcd84c6bc29b718cd2ff4af94", PCI device ID: , UID: "4476546c-b437-48ce-b0bc-2a440047ad63", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0410 16:37:16.431967 3933 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2 b745fb601db8fc8c0068c011db666d813d9ea14dcd84c6bc29b718cd2ff4af94 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-qwqc2 b745fb601db8fc8c0068c011db666d813d9ea14dcd84c6bc29b718cd2ff4af94 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:1c:1a:49:17:ca\",\"name\":\"b745fb601db8fc8\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f876e960-3cce-4653-be4b-e21b27eed98f\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:37:17.965521 3933 obj_retry.go:464] Detected object openshift-console/console-5f495d9d9c-8c9b4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:17.966025 3933 obj_retry.go:464] Detected object openshift-console/console-5f495d9d9c-8c9b4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0410 16:37:18.027781 3933 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0410 16:37:18.027805 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0410 16:37:18.027863 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0410 16:37:18.784555 3933 obj_retry.go:458] Detected object openshift-console/console-5f495d9d9c-8c9b4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:18.784576 3933 obj_retry.go:458] Detected object openshift-console/console-5f495d9d9c-8c9b4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:18.794837 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f495d9d9c-8c9b4 I0410 16:37:18.794863 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f495d9d9c-8c9b4 W0410 16:37:19.025774 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0410 16:37:21.039559 3933 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0410 16:37:27.035501 3933 cni.go:403] [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default] DEL starting CNI request [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default] I0410 16:37:27.112282 3933 cni.go:424] [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default] DEL finished CNI request [openshift-console/console-5f495d9d9c-tq25g 73534e9c8b1fffadfd13c0ecb5007a8b5d181098b1da7654de31631c663ebf24 network default NAD default], result "{}", err I0410 16:37:27.171893 3933 obj_retry.go:464] Detected object openshift-console/console-5f495d9d9c-tq25g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:27.171940 3933 pods.go:176] Deleting pod: openshift-console/console-5f495d9d9c-tq25g I0410 16:37:27.173323 3933 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5f495d9d9c-tq25g, ips: 10.128.0.26 I0410 16:37:27.173356 3933 obj_retry.go:464] Detected object openshift-console/console-5f495d9d9c-tq25g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:28.062954 3933 obj_retry.go:458] Detected object openshift-console/console-5f495d9d9c-tq25g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:28.062980 3933 obj_retry.go:458] Detected object openshift-console/console-5f495d9d9c-tq25g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:28.069670 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f495d9d9c-tq25g I0410 16:37:28.069694 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f495d9d9c-tq25g I0410 16:37:30.222451 3933 cni.go:403] [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default] DEL starting CNI request [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default] I0410 16:37:30.300403 3933 cni.go:424] [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default] DEL finished CNI request [openshift-ingress/router-default-5bd7bdf744-866ds 418ec18095addfc6ffa407d96d297cf16cffd56abae4389ffd0538e1eeb473de network default NAD default], result "{}", err I0410 16:37:30.379408 3933 obj_retry.go:464] Detected object openshift-ingress/router-default-5bd7bdf744-866ds of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:30.379438 3933 pods.go:176] Deleting pod: openshift-ingress/router-default-5bd7bdf744-866ds I0410 16:37:30.380694 3933 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5bd7bdf744-866ds, ips: 10.128.0.12 I0410 16:37:30.380728 3933 obj_retry.go:464] Detected object openshift-ingress/router-default-5bd7bdf744-866ds of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:31.079858 3933 obj_retry.go:458] Detected object openshift-ingress/router-default-5bd7bdf744-866ds of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:31.079884 3933 obj_retry.go:458] Detected object openshift-ingress/router-default-5bd7bdf744-866ds of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:31.089601 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5bd7bdf744-866ds I0410 16:37:31.089633 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5bd7bdf744-866ds W0410 16:37:32.046739 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0410 16:37:49.666597 3933 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:49.667171 3933 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:49.997099 3933 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:49.997129 3933 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:50.012541 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0410 16:37:50.012571 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0410 16:37:52.751145 3933 base_network_controller_pods.go:487] [default/openshift-console/console-6dbc4f68f-29t6d] creating logical port openshift-console_console-6dbc4f68f-29t6d for pod on switch ip-10-0-2-187.ec2.internal I0410 16:37:52.751268 3933 kube.go:256] Updating pod openshift-console/console-6dbc4f68f-29t6d I0410 16:37:52.769265 3933 pod.go:62] [openshift-console/console-6dbc4f68f-29t6d] pod update took 18.033966ms I0410 16:37:52.769286 3933 base_network_controller_pods.go:937] [default/openshift-console/console-6dbc4f68f-29t6d] addLogicalPort annotation time took 18.06149ms I0410 16:37:52.770415 3933 pods.go:271] [openshift-console/console-6dbc4f68f-29t6d] addLogicalPort took 19.284387ms, libovsdb time 774.159µs I0410 16:37:53.114072 3933 cni.go:403] [openshift-console/console-6dbc4f68f-29t6d 537a23b1bf3954a99fd687c919bfe05d918e993e45a275a3e48ad998b44990cd network default NAD default] ADD starting CNI request [openshift-console/console-6dbc4f68f-29t6d 537a23b1bf3954a99fd687c919bfe05d918e993e45a275a3e48ad998b44990cd network default NAD default] I0410 16:37:53.123736 3933 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6dbc4f68f-29t6d, hostIfaceName: 537a23b1bf3954a, network: default, NAD default, SandboxID: "537a23b1bf3954a99fd687c919bfe05d918e993e45a275a3e48ad998b44990cd", PCI device ID: , UID: "9c3a16fd-999a-4805-a9ce-5cafc327e84d", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0410 16:37:53.173025 3933 cni.go:424] [openshift-console/console-6dbc4f68f-29t6d 537a23b1bf3954a99fd687c919bfe05d918e993e45a275a3e48ad998b44990cd network default NAD default] ADD finished CNI request [openshift-console/console-6dbc4f68f-29t6d 537a23b1bf3954a99fd687c919bfe05d918e993e45a275a3e48ad998b44990cd network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:df:f1:19:84:27\",\"name\":\"537a23b1bf3954a\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2dc3fd6-41fe-4ced-8aa8-430caf463cca\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0410 16:37:54.001496 3933 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:54.002060 3933 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:37:54.325592 3933 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:54.325612 3933 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:37:54.333287 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0410 16:37:54.333314 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0410 16:38:03.250275 3933 util.go:959] Failed to build node endpoints for node ip-10-0-2-187.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0410 16:38:18.135770 3933 obj_retry.go:464] Detected object openshift-console/console-864b6677d4-hj2td of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:18.136470 3933 obj_retry.go:464] Detected object openshift-console/console-864b6677d4-hj2td of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:19.044124 3933 obj_retry.go:458] Detected object openshift-console/console-864b6677d4-hj2td of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:19.044177 3933 obj_retry.go:458] Detected object openshift-console/console-864b6677d4-hj2td of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:19.049770 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-864b6677d4-hj2td I0410 16:38:19.049801 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-864b6677d4-hj2td I0410 16:38:21.758894 3933 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:21.759610 3933 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:22.048836 3933 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:22.048862 3933 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:22.056088 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0410 16:38:22.056109 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0410 16:38:28.584845 3933 obj_retry.go:464] Detected object openshift-console/console-864b6677d4-mgfj2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:28.585478 3933 obj_retry.go:464] Detected object openshift-console/console-864b6677d4-mgfj2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:29.173377 3933 obj_retry.go:458] Detected object openshift-console/console-864b6677d4-mgfj2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:29.173398 3933 obj_retry.go:458] Detected object openshift-console/console-864b6677d4-mgfj2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:29.186847 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-864b6677d4-mgfj2 I0410 16:38:29.186871 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-864b6677d4-mgfj2 I0410 16:38:56.616760 3933 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:56.617492 3933 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:38:57.202727 3933 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:57.202753 3933 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:38:57.209586 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0410 16:38:57.209615 3933 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0410 16:40:04.671030 3933 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29597320-zjsgg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0410 16:40:04.671570 3933 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29597320-zjsgg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0410 16:40:05.558861 3933 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29597320-zjsgg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:40:05.558883 3933 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29597320-zjsgg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0410 16:40:57.682672 3933 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0410 16:40:58.067916 3933 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace