+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-172.ec2.internal ++ [[ -n ip-10-0-0-172.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-172.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0423 20:43:13.608174579 - disable conntrack on geneve port' I0423 20:43:13.608174579 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0423 20:43:13.647518851 - starting ovnkube-node + echo 'I0423 20:43:13.647518851 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-172.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-172.ec2.internal --init-node ip-10-0-0-172.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-172.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0423 20:43:13.691332 3796 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0423 20:43:13.691403 3796 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-64c91c14e1.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0423 20:43:13.692825 3796 kube.go:431] Waiting for certificate I0423 20:43:13.692851 3796 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0423 20:43:13.692887 3796 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0423 20:43:13.693607 3796 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 20:43:13.731724 3796 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:13.731738 3796 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:13.738111 3796 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:13.738196 3796 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-2mgkr" I0423 20:43:13.742361 3796 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-2mgkr" I0423 20:43:13.742848 3796 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:14.693360 3796 kube.go:438] Certificate found I0423 20:43:14.693943 3796 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 20:43:14.695065 3796 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0423 20:43:14.747150 3796 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-24 20:38:13 +0000 UTC" deadline="2026-04-24 14:54:15.744924464 +0000 UTC" I0423 20:43:14.747228 3796 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h11m0.997709884s" I0423 20:43:14.778140 3796 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 20:43:14.785780 3796 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0423 20:43:14.785934 3796 server.go:101] MetricServer registers OVS metrics I0423 20:43:14.786440 3796 server.go:105] MetricServer registers OVN DB metrics I0423 20:43:14.787243 3796 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0423 20:43:14.787312 3796 factory.go:561] Starting watch factory I0423 20:43:14.787456 3796 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.787484 3796 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.787440 3796 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.787512 3796 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.787533 3796 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.787540 3796 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.788132 3796 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.788214 3796 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.788653 3796 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.788754 3796 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.788868 3796 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.788969 3796 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.791755 3796 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0423 20:43:14.791779 3796 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0423 20:43:14.793614 3796 controller_manager.go:370] Starting the ovnkube controller I0423 20:43:14.793627 3796 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-172.ec2.internal I0423 20:43:14.793674 3796 controller_manager.go:395] NBDB zone sync took: 38.179µs I0423 20:43:14.793685 3796 factory.go:561] Starting watch factory I0423 20:43:14.805974 3796 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0423 20:43:14.805999 3796 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0423 20:43:14.807155 3796 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.809319 3796 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.809800 3796 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.810647 3796 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0423 20:43:14.810698 3796 server.go:109] MetricServer registers OVN Controller metrics I0423 20:43:14.811758 3796 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.813812 3796 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.815694 3796 server.go:113] MetricServer registers OVN Northd metrics I0423 20:43:14.818045 3796 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:14.820286 3796 metrics.go:519] OVN Metrics Server starts to run ... I0423 20:43:14.893572 3796 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 20:43:14.893607 3796 factory.go:1977] *v1.Node informer cache synced successfully I0423 20:43:14.893613 3796 factory.go:1977] *v1.Service informer cache synced successfully I0423 20:43:14.893617 3796 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 20:43:14.893623 3796 factory.go:1977] *v1.Pod informer cache synced successfully I0423 20:43:14.893627 3796 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 20:43:14.893714 3796 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:14.893732 3796 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:14.893718 3796 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:14.893768 3796 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:14.896930 3796 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:14.896947 3796 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:14.898808 3796 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 20:43:14.898819 3796 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 20:43:14.898828 3796 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 20:43:14.898822 3796 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 20:43:14.898897 3796 factory.go:1977] *v1.Node informer cache synced successfully I0423 20:43:14.898905 3796 factory.go:1977] *v1.Service informer cache synced successfully I0423 20:43:14.898909 3796 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 20:43:14.898912 3796 factory.go:1977] *v1.Pod informer cache synced successfully I0423 20:43:14.898884 3796 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.898933 3796 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.901238 3796 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.904332 3796 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 20:43:14.904345 3796 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 20:43:14.904352 3796 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 20:43:14.904404 3796 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.904415 3796 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.906819 3796 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.909888 3796 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 20:43:14.909962 3796 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.909974 3796 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 20:43:14.909980 3796 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.912342 3796 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.915394 3796 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 20:43:14.915449 3796 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.915460 3796 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.915559 3796 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 20:43:14.917758 3796 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.920932 3796 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 20:43:14.921010 3796 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 20:43:14.921077 3796 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.921093 3796 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.923968 3796 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.926151 3796 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 20:43:14.926162 3796 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 20:43:14.926430 3796 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 20:43:14.926448 3796 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 20:43:14.929017 3796 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 20:43:14.932218 3796 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 20:43:14.932299 3796 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 20:43:14.932393 3796 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.932431 3796 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.932436 3796 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.932450 3796 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.935018 3796 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.935074 3796 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:14.937259 3796 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 20:43:14.937413 3796 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 20:43:14.937423 3796 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 20:43:14.937430 3796 factory.go:680] Watch Factory start up complete, took: 150.12299ms I0423 20:43:14.937492 3796 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0423 20:43:14.943248 3796 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 20:43:14.943260 3796 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 20:43:14.943268 3796 factory.go:680] Watch Factory start up complete, took: 149.581569ms I0423 20:43:14.943275 3796 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-172.ec2.internal" zone I0423 20:43:14.980832 3796 default_node_network_controller.go:710] Initializing the default node network controller I0423 20:43:15.065642 3796 udn_isolation.go:101] Starting UDN host isolation manager I0423 20:43:15.073161 3796 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0423 20:43:15.142238 3796 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0423 20:43:15.142306 3796 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0423 20:43:15.142323 3796 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0423 20:43:15.175450 3796 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0423 20:43:15.175514 3796 default_node_network_controller.go:809] Node ip-10-0-0-172.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0423 20:43:15.180727 3796 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.172"] k8s.ovn.org/zone-name:ip-10-0-0-172.ec2.internal] on node ip-10-0-0-172.ec2.internal I0423 20:43:15.204328 3796 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0423 20:43:15.210222 3796 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0423 20:43:15.210351 3796 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0423 20:43:15.210493 3796 gateway_init.go:263] Preparing Gateway I0423 20:43:15.210505 3796 gateway_shared_intf.go:1687] Creating new gateway I0423 20:43:15.321291 3796 iptables.go:108] Creating table: filter chain: FORWARD I0423 20:43:15.328077 3796 iptables.go:108] Creating table: filter chain: OUTPUT I0423 20:43:15.334790 3796 gateway_shared_intf.go:1816] Gateway Creation Complete I0423 20:43:15.334804 3796 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-172.ec2.internal","mac-address":"12:d3:ec:26:86:c3","ip-addresses":["10.0.0.172/24"],"ip-address":"10.0.0.172/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:8db6fbfa-03f7-46fc-8011-5e1d30e52299 k8s.ovn.org/node-encap-ips:["10.0.0.172"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.172/24"} k8s.ovn.org/zone-name:ip-10-0-0-172.ec2.internal] on node ip-10-0-0-172.ec2.internal I0423 20:43:15.355570 3796 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:15.443672 3796 controller_manager.go:429] Waiting for node in zone sync took: 500.378912ms I0423 20:43:15.448779 3796 controller_manager.go:611] Switching to transit router for layer2 networks I0423 20:43:15.448796 3796 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-172.ec2.internal I0423 20:43:15.465596 3796 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 20:43:15.472553 3796 controller_manager.go:290] SCTP support detected in OVN I0423 20:43:15.473149 3796 services_controller.go:69] Creating services controller for network=default I0423 20:43:15.473215 3796 udn_enabled_svc.go:80] Waiting for service informer to sync I0423 20:43:15.473309 3796 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0423 20:43:15.473339 3796 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0423 20:43:15.473354 3796 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0423 20:43:15.473365 3796 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 20:43:15.473548 3796 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0423 20:43:15.473591 3796 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0423 20:43:15.473672 3796 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 92.763µs I0423 20:43:15.473692 3796 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 93.219µs I0423 20:43:15.473699 3796 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 107.457µs I0423 20:43:15.473704 3796 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0423 20:43:15.473720 3796 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0423 20:43:15.473728 3796 udn_enabled_svc.go:95] Performing full resync I0423 20:43:15.473767 3796 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 5.889µs I0423 20:43:15.473731 3796 default_network_controller.go:325] Starting the default network controller I0423 20:43:15.474090 3796 udn_enabled_svc.go:99] Waiting for handler to sync I0423 20:43:15.474101 3796 udn_enabled_svc.go:104] Starting worker I0423 20:43:15.483810 3796 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0423 20:43:15.483882 3796 repair.go:33] Syncing exgw routes took 56.055µs I0423 20:43:15.483914 3796 default_network_controller.go:414] Starting all the Watchers... I0423 20:43:15.484089 3796 namespace.go:98] [openshift-kube-apiserver] adding namespace I0423 20:43:15.484090 3796 namespace.go:98] [kube-node-lease] adding namespace I0423 20:43:15.484101 3796 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0423 20:43:15.484090 3796 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0423 20:43:15.484105 3796 namespace.go:98] [openshift-service-ca-operator] adding namespace I0423 20:43:15.484112 3796 namespace.go:98] [openshift-apiserver] adding namespace I0423 20:43:15.484116 3796 namespace.go:98] [openshift-network-diagnostics] adding namespace I0423 20:43:15.484127 3796 namespace.go:98] [openshift-operators-redhat] adding namespace I0423 20:43:15.484132 3796 namespace.go:98] [openshift-dns-operator] adding namespace I0423 20:43:15.484135 3796 namespace.go:98] [openshift-dns] adding namespace I0423 20:43:15.484133 3796 namespace.go:98] [openshift-infra] adding namespace I0423 20:43:15.484139 3796 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0423 20:43:15.484139 3796 namespace.go:98] [openshift-multus] adding namespace I0423 20:43:15.484157 3796 namespace.go:98] [openshift-config] adding namespace I0423 20:43:15.484190 3796 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0423 20:43:15.484689 3796 namespace.go:102] [openshift-kube-apiserver] adding namespace took 587.162µs I0423 20:43:15.484704 3796 namespace.go:98] [openshift-kube-scheduler] adding namespace I0423 20:43:15.485170 3796 namespace.go:102] [openshift-kube-scheduler] adding namespace took 460.784µs I0423 20:43:15.485185 3796 namespace.go:98] [openshift-marketplace] adding namespace I0423 20:43:15.485694 3796 namespace.go:102] [openshift-marketplace] adding namespace took 504.288µs I0423 20:43:15.485708 3796 namespace.go:98] [openshift-package-operator] adding namespace I0423 20:43:15.486155 3796 namespace.go:102] [kube-node-lease] adding namespace took 2.048585ms I0423 20:43:15.486164 3796 namespace.go:98] [openshift-machine-config-operator] adding namespace I0423 20:43:15.486647 3796 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 2.53912ms I0423 20:43:15.486655 3796 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0423 20:43:15.487163 3796 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 3.048763ms I0423 20:43:15.487180 3796 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0423 20:43:15.487629 3796 namespace.go:102] [openshift-service-ca-operator] adding namespace took 3.511088ms I0423 20:43:15.487648 3796 namespace.go:98] [openshift-ingress-operator] adding namespace W0423 20:43:15.488030 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-94wvc: pod openshift-network-diagnostics/network-check-target-94wvc: no pod IPs found W0423 20:43:15.488043 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-lnwp6: pod openshift-network-diagnostics/network-check-target-lnwp6: no pod IPs found W0423 20:43:15.488049 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rw6vl: pod openshift-network-diagnostics/network-check-target-rw6vl: no pod IPs found I0423 20:43:15.488070 3796 namespace.go:102] [openshift-apiserver] adding namespace took 3.948932ms I0423 20:43:15.488084 3796 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0423 20:43:15.488531 3796 namespace.go:102] [openshift-network-diagnostics] adding namespace took 4.407796ms I0423 20:43:15.488544 3796 namespace.go:98] [openshift-backplane-mobb] adding namespace I0423 20:43:15.488957 3796 namespace.go:102] [openshift-dns-operator] adding namespace took 4.820495ms I0423 20:43:15.488965 3796 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0423 20:43:15.489357 3796 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.222876ms I0423 20:43:15.489419 3796 namespace.go:98] [openshift-node] adding namespace I0423 20:43:15.489775 3796 namespace.go:102] [openshift-dns] adding namespace took 5.635996ms I0423 20:43:15.489788 3796 namespace.go:98] [openshift-operators] adding namespace I0423 20:43:15.490175 3796 namespace.go:102] [openshift-infra] adding namespace took 6.034108ms I0423 20:43:15.490187 3796 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace W0423 20:43:15.490560 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-47kfb: pod openshift-multus/network-metrics-daemon-47kfb: no pod IPs found W0423 20:43:15.490570 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-khgmn: pod openshift-multus/network-metrics-daemon-khgmn: no pod IPs found W0423 20:43:15.490576 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-f2fwk: pod openshift-multus/network-metrics-daemon-f2fwk: no pod IPs found I0423 20:43:15.490603 3796 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.460008ms I0423 20:43:15.490616 3796 namespace.go:98] [openshift-backplane-cse] adding namespace I0423 20:43:15.491017 3796 namespace.go:102] [openshift-multus] adding namespace took 6.869471ms I0423 20:43:15.491031 3796 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0423 20:43:15.491472 3796 namespace.go:102] [openshift-config] adding namespace took 7.306764ms I0423 20:43:15.491482 3796 namespace.go:98] [openshift-config-managed] adding namespace I0423 20:43:15.491860 3796 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.658341ms I0423 20:43:15.491876 3796 namespace.go:98] [openshift-backplane-tam] adding namespace I0423 20:43:15.492242 3796 namespace.go:102] [openshift-package-operator] adding namespace took 6.527247ms I0423 20:43:15.492253 3796 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0423 20:43:15.492642 3796 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.472152ms I0423 20:43:15.492658 3796 namespace.go:98] [openshift-insights] adding namespace I0423 20:43:15.493025 3796 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.364686ms I0423 20:43:15.493036 3796 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0423 20:43:15.493393 3796 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.208184ms I0423 20:43:15.493403 3796 namespace.go:98] [kube-system] adding namespace I0423 20:43:15.493825 3796 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.170921ms I0423 20:43:15.493839 3796 namespace.go:98] [dedicated-admin] adding namespace I0423 20:43:15.494186 3796 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.097244ms I0423 20:43:15.494198 3796 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0423 20:43:15.494575 3796 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.026277ms I0423 20:43:15.494592 3796 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0423 20:43:15.494946 3796 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.977248ms I0423 20:43:15.494958 3796 namespace.go:98] [openshift-backplane-srep] adding namespace I0423 20:43:15.495350 3796 namespace.go:102] [openshift-node] adding namespace took 5.92077ms I0423 20:43:15.495365 3796 namespace.go:98] [openshift-network-operator] adding namespace I0423 20:43:15.495733 3796 namespace.go:102] [openshift-operators] adding namespace took 5.940171ms I0423 20:43:15.495745 3796 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0423 20:43:15.496117 3796 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 5.925566ms I0423 20:43:15.496140 3796 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0423 20:43:15.496526 3796 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.904503ms I0423 20:43:15.496543 3796 namespace.go:98] [openshift-ingress] adding namespace I0423 20:43:15.496917 3796 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.881541ms I0423 20:43:15.496931 3796 namespace.go:98] [openshift-host-network] adding namespace I0423 20:43:15.497291 3796 namespace.go:102] [openshift-config-managed] adding namespace took 5.804357ms I0423 20:43:15.497304 3796 namespace.go:98] [openshift-backplane] adding namespace I0423 20:43:15.497675 3796 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.793777ms I0423 20:43:15.497690 3796 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0423 20:43:15.498032 3796 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 5.77454ms I0423 20:43:15.498046 3796 namespace.go:98] [openshift-monitoring] adding namespace I0423 20:43:15.498412 3796 namespace.go:102] [openshift-insights] adding namespace took 5.74945ms I0423 20:43:15.498424 3796 namespace.go:98] [openshift-config-operator] adding namespace I0423 20:43:15.498809 3796 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.767561ms I0423 20:43:15.498822 3796 namespace.go:98] [openshift-authentication-operator] adding namespace I0423 20:43:15.499563 3796 namespace.go:102] [kube-system] adding namespace took 6.155236ms I0423 20:43:15.499577 3796 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0423 20:43:15.499947 3796 namespace.go:102] [dedicated-admin] adding namespace took 6.103161ms I0423 20:43:15.499957 3796 namespace.go:98] [openshift-console-user-settings] adding namespace I0423 20:43:15.500311 3796 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.107493ms I0423 20:43:15.500323 3796 namespace.go:98] [openshift] adding namespace I0423 20:43:15.500696 3796 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.098634ms I0423 20:43:15.500707 3796 namespace.go:98] [openshift-backplane-cee] adding namespace I0423 20:43:15.501053 3796 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.090274ms I0423 20:43:15.501063 3796 namespace.go:98] [openshift-network-console] adding namespace I0423 20:43:15.501419 3796 namespace.go:102] [openshift-network-operator] adding namespace took 6.030642ms I0423 20:43:15.501431 3796 namespace.go:98] [openshift-machine-api] adding namespace I0423 20:43:15.501797 3796 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.047632ms I0423 20:43:15.501808 3796 namespace.go:98] [openshift-authentication] adding namespace I0423 20:43:15.502164 3796 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.018761ms I0423 20:43:15.502174 3796 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0423 20:43:15.502573 3796 namespace.go:102] [openshift-ingress] adding namespace took 6.02458ms I0423 20:43:15.502584 3796 namespace.go:98] [openshift-apiserver-operator] adding namespace I0423 20:43:15.502977 3796 namespace.go:102] [openshift-host-network] adding namespace took 6.039382ms I0423 20:43:15.502991 3796 namespace.go:98] [openshift-etcd] adding namespace I0423 20:43:15.503340 3796 namespace.go:102] [openshift-backplane] adding namespace took 6.032935ms I0423 20:43:15.503350 3796 namespace.go:98] [openshift-image-registry] adding namespace I0423 20:43:15.503748 3796 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.051911ms I0423 20:43:15.503763 3796 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0423 20:43:15.504106 3796 namespace.go:102] [openshift-monitoring] adding namespace took 6.053866ms I0423 20:43:15.504482 3796 namespace.go:102] [openshift-config-operator] adding namespace took 6.052285ms I0423 20:43:15.504496 3796 namespace.go:98] [default] adding namespace I0423 20:43:15.504842 3796 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.015883ms I0423 20:43:15.504854 3796 namespace.go:98] [openshift-route-controller-manager] adding namespace I0423 20:43:15.505225 3796 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.644332ms I0423 20:43:15.505240 3796 namespace.go:98] [openshift-cluster-version] adding namespace I0423 20:43:15.505647 3796 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.685145ms I0423 20:43:15.505660 3796 namespace.go:98] [openshift-backplane-csm] adding namespace I0423 20:43:15.505994 3796 namespace.go:102] [openshift] adding namespace took 5.664733ms I0423 20:43:15.506008 3796 namespace.go:98] [openshift-logging] adding namespace I0423 20:43:15.506403 3796 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.690807ms I0423 20:43:15.506417 3796 namespace.go:98] [openshift-ingress-canary] adding namespace I0423 20:43:15.506779 3796 namespace.go:102] [openshift-network-console] adding namespace took 5.71038ms I0423 20:43:15.506792 3796 namespace.go:98] [openshift-customer-monitoring] adding namespace I0423 20:43:15.507220 3796 namespace.go:102] [openshift-machine-api] adding namespace took 5.784188ms I0423 20:43:15.507233 3796 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0423 20:43:15.507610 3796 namespace.go:102] [openshift-authentication] adding namespace took 5.798744ms I0423 20:43:15.507620 3796 namespace.go:98] [openshift-console] adding namespace I0423 20:43:15.507976 3796 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 5.794657ms I0423 20:43:15.507991 3796 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0423 20:43:15.508322 3796 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.734112ms I0423 20:43:15.508332 3796 namespace.go:98] [openshift-network-node-identity] adding namespace I0423 20:43:15.508686 3796 namespace.go:102] [openshift-etcd] adding namespace took 5.68969ms I0423 20:43:15.508696 3796 namespace.go:98] [open-cluster-management-2ps35p4e7fo08ot66mbftq9uvphot0ei] adding namespace I0423 20:43:15.509060 3796 namespace.go:102] [openshift-image-registry] adding namespace took 5.705052ms I0423 20:43:15.509074 3796 namespace.go:98] [kube-public] adding namespace I0423 20:43:15.509438 3796 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.669817ms I0423 20:43:15.509453 3796 namespace.go:98] [openshift-controller-manager] adding namespace I0423 20:43:15.509804 3796 namespace.go:102] [default] adding namespace took 5.302524ms I0423 20:43:15.510178 3796 namespace.go:102] [openshift-route-controller-manager] adding namespace took 5.319838ms I0423 20:43:15.510188 3796 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0423 20:43:15.510590 3796 namespace.go:102] [openshift-cluster-version] adding namespace took 5.345363ms I0423 20:43:15.510973 3796 namespace.go:102] [openshift-backplane-csm] adding namespace took 5.306369ms I0423 20:43:15.511331 3796 namespace.go:102] [openshift-logging] adding namespace took 5.319376ms I0423 20:43:15.511342 3796 namespace.go:98] [openshift-must-gather-operator] adding namespace I0423 20:43:15.511709 3796 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.287069ms I0423 20:43:15.512085 3796 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.288474ms I0423 20:43:15.512447 3796 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 5.210789ms I0423 20:43:15.512873 3796 namespace.go:102] [openshift-console] adding namespace took 5.247215ms I0423 20:43:15.513245 3796 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.24694ms I0423 20:43:15.513655 3796 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.31808ms I0423 20:43:15.513667 3796 namespace.go:98] [openshift-console-operator] adding namespace I0423 20:43:15.513996 3796 namespace.go:102] [open-cluster-management-2ps35p4e7fo08ot66mbftq9uvphot0ei] adding namespace took 5.294903ms I0423 20:43:15.514454 3796 namespace.go:102] [kube-public] adding namespace took 5.374866ms I0423 20:43:15.514822 3796 namespace.go:102] [openshift-controller-manager] adding namespace took 5.363409ms I0423 20:43:15.515197 3796 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 5.004026ms I0423 20:43:15.515549 3796 namespace.go:102] [openshift-must-gather-operator] adding namespace took 4.201894ms I0423 20:43:15.515920 3796 namespace.go:102] [openshift-console-operator] adding namespace took 2.249323ms W0423 20:43:15.515987 3796 master.go:471] Unable to parse remote node ip-10-0-1-197.ec2.internal chassis-id annotation. Chassis may be removed during sync W0423 20:43:15.515995 3796 master.go:471] Unable to parse remote node ip-10-0-2-22.ec2.internal chassis-id annotation. Chassis may be removed during sync I0423 20:43:15.516065 3796 master.go:548] Adding or Updating local node "ip-10-0-0-172.ec2.internal" for network "default" I0423 20:43:15.516134 3796 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-22.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-22.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-22.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-22.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-22.ec2.internal I0423 20:43:15.516133 3796 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-197.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-197.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-197.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-197.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-197.ec2.internal W0423 20:43:15.523991 3796 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-94wvc: pod openshift-network-diagnostics/network-check-target-94wvc: no pod IPs found W0423 20:43:15.524006 3796 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-47kfb: pod openshift-multus/network-metrics-daemon-47kfb: no pod IPs found I0423 20:43:15.524948 3796 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-172.ec2.internal for the network default I0423 20:43:15.525298 3796 zone_ic_handler.go:195] Time taken to create transit switch: 337.764µs I0423 20:43:15.526134 3796 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0423 20:43:15.526159 3796 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0423 20:43:15.526173 3796 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0423 20:43:15.526179 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-172.ec2.internal in network "default" I0423 20:43:15.526227 3796 node_tracker.go:169] Node ip-10-0-0-172.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:15.526243 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:15.526250 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-197.ec2.internal in network "default" I0423 20:43:15.526258 3796 node_tracker.go:235] Node ip-10-0-1-197.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-197.ec2.internal" I0423 20:43:15.526265 3796 node_tracker.go:169] Node ip-10-0-1-197.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:15.526269 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:15.526272 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0423 20:43:15.526279 3796 node_tracker.go:235] Node ip-10-0-2-22.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-22.ec2.internal" I0423 20:43:15.526284 3796 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:15.526287 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:15.626823 3796 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0423 20:43:15.626844 3796 services_controller.go:208] Setting up event handlers for services for network=default I0423 20:43:15.626963 3796 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0423 20:43:15.627011 3796 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0423 20:43:15.627032 3796 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0423 20:43:15.727315 3796 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0423 20:43:15.727334 3796 repair.go:60] Starting repairing loop for services I0423 20:43:15.727442 3796 repair.go:131] Deleted 0 stale service LBs I0423 20:43:15.727462 3796 repair.go:137] Deleted 0 stale Chassis Template Vars I0423 20:43:15.727477 3796 repair.go:62] Finished repairing loop for services: 145.196µs I0423 20:43:15.727503 3796 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0423 20:43:15.727513 3796 services_controller.go:254] Starting workers for network=default I0423 20:43:15.727844 3796 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-94wvc] creating logical port openshift-network-diagnostics_network-check-target-94wvc for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:15.727868 3796 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-47kfb] creating logical port openshift-multus_network-metrics-daemon-47kfb for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:15.727868 3796 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rw6vl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rw6vl: no pod IPs found I0423 20:43:15.727892 3796 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-khgmn, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-khgmn: suppressed error logged: pod openshift-multus/network-metrics-daemon-khgmn: no pod IPs found I0423 20:43:15.727905 3796 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-f2fwk, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-f2fwk: suppressed error logged: pod openshift-multus/network-metrics-daemon-f2fwk: no pod IPs found I0423 20:43:15.727933 3796 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-lnwp6, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-lnwp6: suppressed error logged: pod openshift-network-diagnostics/network-check-target-lnwp6: no pod IPs found I0423 20:43:15.727997 3796 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-94wvc I0423 20:43:15.728072 3796 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-47kfb I0423 20:43:15.744642 3796 pod.go:62] [openshift-network-diagnostics/network-check-target-94wvc] pod update took 16.727498ms I0423 20:43:15.744658 3796 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-94wvc] addLogicalPort annotation time took 16.750884ms I0423 20:43:15.746041 3796 pods.go:271] [openshift-network-diagnostics/network-check-target-94wvc] addLogicalPort took 18.202594ms, libovsdb time 1.097987ms I0423 20:43:15.747601 3796 pod.go:62] [openshift-multus/network-metrics-daemon-47kfb] pod update took 19.666551ms I0423 20:43:15.747656 3796 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-47kfb] addLogicalPort annotation time took 19.7202ms I0423 20:43:15.748582 3796 pods.go:271] [openshift-multus/network-metrics-daemon-47kfb] addLogicalPort took 20.727147ms, libovsdb time 690.802µs I0423 20:43:15.749262 3796 admin_network_policy_controller.go:227] Starting controller default-network-controller I0423 20:43:15.749287 3796 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0423 20:43:15.749302 3796 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0423 20:43:15.749310 3796 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0423 20:43:15.749525 3796 repair.go:29] Repairing admin network policies took 209.288µs I0423 20:43:15.749693 3796 repair.go:92] Repairing baseline admin network policies took 155.898µs I0423 20:43:15.750150 3796 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0423 20:43:15.750169 3796 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0423 20:43:15.751595 3796 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0423 20:43:15.751620 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0423 20:43:15.751647 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0423 20:43:15.753869 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0423 20:43:15.753894 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0423 20:43:15.753909 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0423 20:43:15.755173 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0423 20:43:15.755192 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0423 20:43:15.755201 3796 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0423 20:43:15.757045 3796 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0423 20:43:15.757068 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0423 20:43:15.757472 3796 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0423 20:43:15.757490 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 20:43:15.759294 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0423 20:43:15.759319 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0423 20:43:15.759327 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0423 20:43:15.761019 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0423 20:43:15.761042 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0423 20:43:15.761049 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0423 20:43:15.761335 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0423 20:43:15.761355 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0423 20:43:15.761365 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0423 20:43:15.763408 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0423 20:43:15.763425 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0423 20:43:15.763435 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0423 20:43:15.763989 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0423 20:43:15.764015 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0423 20:43:15.764022 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0423 20:43:15.765395 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0423 20:43:15.765417 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0423 20:43:15.765428 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0423 20:43:15.767059 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0423 20:43:15.767079 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0423 20:43:15.767090 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0423 20:43:15.768084 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0423 20:43:15.768106 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0423 20:43:15.768118 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0423 20:43:15.768635 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0423 20:43:15.768653 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0423 20:43:15.768660 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0423 20:43:15.769205 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0423 20:43:15.769230 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0423 20:43:15.769248 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0423 20:43:15.769808 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0423 20:43:15.769823 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0423 20:43:15.769828 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0423 20:43:15.770091 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0423 20:43:15.770108 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0423 20:43:15.770115 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0423 20:43:15.770647 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0423 20:43:15.770672 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0423 20:43:15.770683 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0423 20:43:15.771658 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0423 20:43:15.771683 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0423 20:43:15.771700 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0423 20:43:15.772928 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0423 20:43:15.772946 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0423 20:43:15.772953 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0423 20:43:15.774047 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0423 20:43:15.774071 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0423 20:43:15.774081 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0423 20:43:15.774600 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0423 20:43:15.774616 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0423 20:43:15.774623 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0423 20:43:15.775089 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0423 20:43:15.775107 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0423 20:43:15.775126 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 20:43:15.775845 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0423 20:43:15.775861 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0423 20:43:15.775868 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0423 20:43:15.776717 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0423 20:43:15.776741 3796 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0423 20:43:15.776753 3796 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0423 20:43:15.777737 3796 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0423 20:43:15.777756 3796 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0423 20:43:15.777767 3796 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0423 20:43:15.778563 3796 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0423 20:43:15.778580 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0423 20:43:15.778586 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0423 20:43:15.778843 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0423 20:43:15.778865 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0423 20:43:15.778878 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0423 20:43:15.779334 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0423 20:43:15.779352 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0423 20:43:15.779362 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0423 20:43:15.779849 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0423 20:43:15.784106 3796 egressfirewall.go:374] Starting EgressFirewall controller I0423 20:43:15.784137 3796 controller.go:133] Adding controller egress-firewall-controller event handlers I0423 20:43:15.784142 3796 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0423 20:43:15.784169 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0423 20:43:15.784183 3796 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0423 20:43:15.784167 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0423 20:43:15.784190 3796 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0423 20:43:15.784455 3796 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0423 20:43:15.784469 3796 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0423 20:43:15.784475 3796 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0423 20:43:15.784483 3796 egressqos.go:193] Setting up event handlers for EgressQoS I0423 20:43:15.784491 3796 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0423 20:43:15.784501 3796 egressfirewall.go:723] Syncing node "ip-10-0-0-172.ec2.internal" for egress firewall I0423 20:43:15.784505 3796 egressfirewall.go:723] Syncing node "ip-10-0-1-197.ec2.internal" for egress firewall I0423 20:43:15.784934 3796 egressqos.go:245] Starting EgressQoS Controller I0423 20:43:15.784945 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0423 20:43:15.784951 3796 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0423 20:43:15.784958 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0423 20:43:15.784962 3796 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0423 20:43:15.784967 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0423 20:43:15.784971 3796 shared_informer.go:356] "Caches are synced" controller="egressqos" I0423 20:43:15.784975 3796 egressqos.go:259] Repairing EgressQoSes I0423 20:43:15.784978 3796 egressqos.go:399] Starting repairing loop for egressqos I0423 20:43:15.785043 3796 egressqos.go:401] Finished repairing loop for egressqos: 64.342µs I0423 20:43:15.785057 3796 egressservice_zone.go:132] Setting up event handlers for Egress Services I0423 20:43:15.785073 3796 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-172.ec2.internal I0423 20:43:15.785115 3796 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-172.ec2.internal : 41.3µs I0423 20:43:15.785178 3796 egressservice_zone.go:210] Starting Egress Services Controller I0423 20:43:15.785194 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 20:43:15.785206 3796 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 20:43:15.785215 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 20:43:15.785224 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 20:43:15.785233 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 20:43:15.785241 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 20:43:15.785250 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0423 20:43:15.785258 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0423 20:43:15.785266 3796 egressservice_zone.go:228] Repairing Egress Services I0423 20:43:15.785519 3796 master_controller.go:88] Starting Admin Policy Based Route Controller I0423 20:43:15.785528 3796 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 20:43:15.785591 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-172.ec2.internal I0423 20:43:15.785604 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-172.ec2.internal: 13.493µs I0423 20:43:15.785614 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-197.ec2.internal I0423 20:43:15.785615 3796 default_network_controller.go:585] Completing all the Watchers took 301.690106ms I0423 20:43:15.785620 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-197.ec2.internal: 7.14µs I0423 20:43:15.785625 3796 default_network_controller.go:589] Starting unidling controllers I0423 20:43:15.785628 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:15.785634 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 5.991µs I0423 20:43:15.785636 3796 unidle.go:46] Registering OVN SB ControllerEvent handler I0423 20:43:15.785649 3796 unidle.go:63] Populating Initial ContollerEvent events I0423 20:43:15.785666 3796 unidle.go:79] Setting up event handlers for services I0423 20:43:15.785725 3796 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0423 20:43:15.856978 3796 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:16.356828 3796 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:16.388607 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-197.ec2.internal in network "default" I0423 20:43:16.388639 3796 node_tracker.go:235] Node ip-10-0-1-197.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-197.ec2.internal" I0423 20:43:16.388658 3796 node_tracker.go:169] Node ip-10-0-1-197.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:16.388674 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:16.388716 3796 default_network_controller.go:1025] Node "ip-10-0-1-197.ec2.internal" in remote zone "ip-10-0-1-197.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:16.444504 3796 default_network_controller.go:1025] Node "ip-10-0-1-197.ec2.internal" in remote zone "ip-10-0-1-197.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:16.558586 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-197.ec2.internal in network "default" I0423 20:43:16.558595 3796 default_network_controller.go:1025] Node "ip-10-0-1-197.ec2.internal" in remote zone "ip-10-0-1-197.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:16.558654 3796 node_tracker.go:169] Node ip-10-0-1-197.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:16.558666 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:16.559582 3796 zone_ic_handler.go:195] Time taken to create transit switch: 248.868µs I0423 20:43:16.559598 3796 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-197.ec2.internal for the network default I0423 20:43:16.560656 3796 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-197.ec2.internal" on network "default" took: 1.334561ms I0423 20:43:16.857181 3796 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:17.356814 3796 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:17.581821 3796 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0423 20:43:17.857058 3796 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:17.875245 3796 bridgeconfig.go:537] Gateway is ready I0423 20:43:17.875265 3796 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0423 20:43:17.887078 3796 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.172. Updating OVN encap IP. I0423 20:43:17.892652 3796 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.172 - it is already configured I0423 20:43:17.892669 3796 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.172/24:{} 172.20.0.1/32:{}]. Updating annotations. I0423 20:43:17.892934 3796 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.172/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-172.ec2.internal","mac-address":"12:d3:ec:26:86:c3","ip-addresses":["10.0.0.172/24"],"ip-address":"10.0.0.172/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:8db6fbfa-03f7-46fc-8011-5e1d30e52299 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.172/24"}] on node ip-10-0-0-172.ec2.internal I0423 20:43:17.909441 3796 egressfirewall.go:723] Syncing node "ip-10-0-0-172.ec2.internal" for egress firewall I0423 20:43:17.909454 3796 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.172 I0423 20:43:17.909506 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-172.ec2.internal I0423 20:43:17.909543 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-172.ec2.internal in network "default" I0423 20:43:17.909524 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-172.ec2.internal: 23.095µs I0423 20:43:17.909557 3796 master.go:548] Adding or Updating local node "ip-10-0-0-172.ec2.internal" for network "default" I0423 20:43:17.909593 3796 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-172.ec2.internal I0423 20:43:17.909615 3796 node_tracker.go:169] Node ip-10-0-0-172.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:17.909634 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:17.914567 3796 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-172.ec2.internal. Updating no re-route policies I0423 20:43:17.914667 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-2dk8b I0423 20:43:17.914679 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-47kfb I0423 20:43:17.914683 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-r4pbt I0423 20:43:17.914696 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-7qtq5 I0423 20:43:17.914700 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-8rqsj I0423 20:43:17.914704 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-s25tl I0423 20:43:17.914712 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-7qtq5 I0423 20:43:17.914716 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-vmpd8 I0423 20:43:17.914722 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-s25tl I0423 20:43:17.914718 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-tvjpk I0423 20:43:17.914729 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-khgmn I0423 20:43:17.914725 3796 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-7czhc I0423 20:43:17.914736 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-tvjpk I0423 20:43:17.914737 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-khgmn I0423 20:43:17.914734 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-vmpd8 I0423 20:43:17.914747 3796 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-7czhc I0423 20:43:17.914759 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-tvjpk after 0 failed attempt(s) I0423 20:43:17.914770 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-vmpd8 after 0 failed attempt(s) I0423 20:43:17.914769 3796 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-7czhc after 0 failed attempt(s) I0423 20:43:17.914687 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-2dk8b I0423 20:43:17.914811 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-2dk8b after 0 failed attempt(s) I0423 20:43:17.914677 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-dc6p4 I0423 20:43:17.914829 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-dc6p4 I0423 20:43:17.914845 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-dc6p4 after 0 failed attempt(s) I0423 20:43:17.914733 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-47kfb I0423 20:43:17.914850 3796 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-khgmn, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-khgmn: suppressed error logged: pod openshift-multus/network-metrics-daemon-khgmn: no pod IPs found I0423 20:43:17.914856 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-s25tl after 0 failed attempt(s) I0423 20:43:17.914713 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-8rqsj I0423 20:43:17.914884 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-8rqsj after 0 failed attempt(s) I0423 20:43:17.914888 3796 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-47kfb] creating logical port openshift-multus_network-metrics-daemon-47kfb for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:17.914701 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-r4pbt I0423 20:43:17.914905 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-r4pbt after 0 failed attempt(s) I0423 20:43:17.914680 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl I0423 20:43:17.914914 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl I0423 20:43:17.914959 3796 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rw6vl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rw6vl: no pod IPs found I0423 20:43:17.914666 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-172.ec2.internal I0423 20:43:17.914983 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-172.ec2.internal I0423 20:43:17.915003 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-172.ec2.internal after 0 failed attempt(s) I0423 20:43:17.914698 3796 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-172.ec2.internal I0423 20:43:17.915026 3796 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-172.ec2.internal I0423 20:43:17.915045 3796 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-172.ec2.internal after 0 failed attempt(s) I0423 20:43:17.914730 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-94wvc I0423 20:43:17.915059 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-94wvc I0423 20:43:17.915088 3796 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-94wvc] creating logical port openshift-network-diagnostics_network-check-target-94wvc for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:17.914742 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-7qtq5 after 0 failed attempt(s) I0423 20:43:17.915889 3796 pods.go:271] [openshift-multus/network-metrics-daemon-47kfb] addLogicalPort took 1.011173ms, libovsdb time 390.639µs I0423 20:43:17.915903 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-47kfb after 0 failed attempt(s) I0423 20:43:17.915956 3796 pods.go:271] [openshift-network-diagnostics/network-check-target-94wvc] addLogicalPort took 874.902µs, libovsdb time 485.308µs I0423 20:43:17.915971 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-94wvc after 0 failed attempt(s) I0423 20:43:17.925940 3796 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0423 20:43:17.938498 3796 iptables.go:108] Creating table: mangle chain: OUTPUT I0423 20:43:17.941999 3796 iptables.go:108] Creating table: nat chain: OUTPUT I0423 20:43:17.985232 3796 iptables.go:108] Creating table: nat chain: PREROUTING I0423 20:43:18.043801 3796 iptables.go:108] Creating table: filter chain: FORWARD I0423 20:43:18.062428 3796 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0423 20:43:18.062530 3796 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0423 20:43:18.062571 3796 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0423 20:43:18.062651 3796 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0423 20:43:18.062699 3796 gateway_init.go:325] Gateway and management port readiness took 2.710866176s I0423 20:43:18.062725 3796 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0423 20:43:18.062760 3796 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0423 20:43:18.062773 3796 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0423 20:43:18.062782 3796 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 20:43:18.062839 3796 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0423 20:43:18.062930 3796 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 60.437µs I0423 20:43:18.063163 3796 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0423 20:43:18.063188 3796 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.116µs I0423 20:43:18.063196 3796 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.731µs I0423 20:43:18.063202 3796 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0423 20:43:18.063210 3796 nad_controller.go:379] [node-nad-controller NAD controller]: started I0423 20:43:18.063215 3796 default_node_network_controller.go:899] Starting the default node network controller I0423 20:43:18.063219 3796 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0423 20:43:18.063319 3796 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 8.058µs I0423 20:43:18.063419 3796 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0423 20:43:18.063428 3796 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0423 20:43:18.063462 3796 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bee890 0xc0002bfce0 0xc0002bfd50 0xc0002bfdc0 0xc0002bfe30 0xc0002bfea0 0xc0002bff10 0xc000cc85a0 0xc0002dc000 0xc0002dc0e0 0xc000c1e370 0xc0063f3340 0xc0002dc150 map[0x2ff9080:0xc001f7ff20 0x2ff95c0:0xc005386120 0x2ffa2e0:0xc001f7fe60 0x2ffa580:0xc001f7fec0 0x2ffad60:0xc005386000 0x2ffb000:0xc005386060 0x2ffb2a0:0xc0053860c0 0x2ffba80:0xc005386300 0x2ffbd20:0xc005386360 0x3026980:0xc0007c45a0 0x3026d20:0xc0007c4600 0x3027800:0xc000c85680 0x3029160:0xc000c34600 0x30298a0:0xc001f7fe00 0x302a720:0xc0009cfaa0] 0xc00064cbd0 0} I0423 20:43:18.063521 3796 gateway.go:151] Starting gateway service sync I0423 20:43:18.063958 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0423 20:43:18.066354 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0423 20:43:18.087976 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0423 20:43:18.089273 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0423 20:43:18.090506 3796 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0423 20:43:18.123185 3796 gateway.go:168] Gateway service sync done. Time taken: 59.651406ms I0423 20:43:18.123785 3796 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30200 I0423 20:43:18.124018 3796 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bee890 0xc0002bfce0 0xc0002bfd50 0xc0002bfdc0 0xc0002bfe30 0xc0002bfea0 0xc0002bff10 0xc000cc85a0 0xc0002dc000 0xc0002dc0e0 0xc000c1e370 0xc0063f3340 0xc0002dc150 map[0x2ff9080:0xc001f7ff20 0x2ff95c0:0xc005386120 0x2ffa2e0:0xc001f7fe60 0x2ffa580:0xc001f7fec0 0x2ffad60:0xc005386000 0x2ffb000:0xc005386060 0x2ffb2a0:0xc0053860c0 0x2ffba80:0xc005386300 0x2ffbd20:0xc005386360 0x3026980:0xc0007c45a0 0x3026d20:0xc0007c4600 0x3027800:0xc000c85680 0x3029160:0xc000c34600 0x30298a0:0xc001f7fe00 0x302a720:0xc0009cfaa0] 0xc00064cbd0 0} I0423 20:43:18.124510 3796 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bee890 0xc0002bfce0 0xc0002bfd50 0xc0002bfdc0 0xc0002bfe30 0xc0002bfea0 0xc0002bff10 0xc000cc85a0 0xc0002dc000 0xc0002dc0e0 0xc000c1e370 0xc0063f3340 0xc0002dc150 map[0x2ff9080:0xc001f7ff20 0x2ff95c0:0xc005386120 0x2ffa2e0:0xc001f7fe60 0x2ffa580:0xc001f7fec0 0x2ffad60:0xc005386000 0x2ffb000:0xc005386060 0x2ffb2a0:0xc0053860c0 0x2ffba80:0xc005386300 0x2ffbd20:0xc005386360 0x3026980:0xc0007c45a0 0x3026d20:0xc0007c4600 0x3027800:0xc000c85680 0x3029160:0xc000c34600 0x30298a0:0xc001f7fe00 0x302a720:0xc0009cfaa0] 0xc00064cbd0 0} I0423 20:43:18.124551 3796 gateway.go:532] Reconciling gateway with updates I0423 20:43:18.129540 3796 gateway.go:339] Spawning Conntrack Rule Check Thread I0423 20:43:18.129586 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:18.129592 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:18.129616 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:18.129617 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:18.129680 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:18.129693 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:18.129708 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:18.129710 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:18.129719 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:18.129723 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:18.129733 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:18.129736 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:18.129747 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:18.129756 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:18.129758 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 20:43:18.129753 3796 default_node_network_controller.go:944] Gateway and management port readiness took 66.316134ms I0423 20:43:18.129772 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 32482 is already open, no action needed I0423 20:43:18.129774 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:18.129774 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 20:43:18.129712 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:18.129786 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 32393 is already open, no action needed I0423 20:43:18.129787 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 20:43:18.129785 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:18.129798 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:18.129808 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:18.129805 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:18.129852 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 20:43:18.129825 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 20:43:18.129869 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:18.129821 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:18.130021 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:18.130032 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:18.130083 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:18.129841 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 20:43:18.130089 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:18.130102 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:18.130107 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 20:43:18.129784 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:18.130119 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:18.130123 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:18.130129 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:18.129838 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:18.130133 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:18.130146 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 20:43:18.130152 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:18.130160 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 20:43:18.130137 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:18.130129 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 20:43:18.130242 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:18.129762 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:18.130252 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:18.130259 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 20:43:18.129848 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 20:43:18.129836 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:18.130271 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:18.130276 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:18.130286 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:18.130311 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 20:43:18.130351 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:18.130386 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:18.130396 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 20:43:18.130397 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:18.130356 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 20:43:18.130413 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:18.130415 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:18.130423 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 20:43:18.130430 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:18.130336 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:18.130479 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:18.130485 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:18.130489 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 20:43:18.130495 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:18.130527 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 20:43:18.130533 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 20:43:18.130538 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 20:43:18.130551 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:18.130561 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:18.130567 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:18.130584 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:18.130596 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:18.130622 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:18.130682 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:18.130697 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:18.130705 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:18.130720 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:18.130790 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:18.130815 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:18.130855 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:18.130874 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:18.130919 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:18.130935 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:18.131019 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:18.131039 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:18.131041 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:18.131057 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:18.131109 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:18.131126 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:18.131170 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:18.131183 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:18.131184 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:18.131188 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 20:43:18.131199 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:18.131208 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 20:43:18.131226 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:18.131237 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:18.131284 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:18.131295 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:18.131749 3796 node_ip_handler_linux.go:160] Node IP manager is running W0423 20:43:18.174146 3796 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 20:43:18.174351 3796 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 20:43:18.174607 3796 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0423 20:43:18.184747 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 20:43:18.184757 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 20:43:18.184774 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 20:43:18.184779 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 20:43:18.184785 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 20:43:18.184792 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 20:43:18.227640 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 20:43:18.227669 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 20:43:18.227682 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 20:43:18.227690 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 20:43:18.266626 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 20:43:18.266669 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 20:43:18.266676 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 20:43:18.266682 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 20:43:18.266688 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 20:43:18.266693 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 20:43:18.266699 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 20:43:18.266704 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 20:43:18.266709 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 20:43:18.266714 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 20:43:18.266720 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 20:43:18.266724 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 20:43:18.266729 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 20:43:18.266734 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 20:43:18.266739 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 20:43:18.266743 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 20:43:18.266760 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 20:43:18.266779 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 20:43:18.617615 3796 egressfirewall.go:723] Syncing node "ip-10-0-1-197.ec2.internal" for egress firewall I0423 20:43:18.617632 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-197.ec2.internal I0423 20:43:18.617649 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-197.ec2.internal: 20.986µs I0423 20:43:18.617687 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-197.ec2.internal in network "default" I0423 20:43:18.617711 3796 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-197.ec2.internal. Updating no re-route policies I0423 20:43:18.617736 3796 node_tracker.go:169] Node ip-10-0-1-197.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:18.617753 3796 services_controller.go:574] Full service sync requested for network=default I0423 20:43:19.177678 3796 default_network_controller.go:1025] Node "ip-10-0-2-22.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:20.114208 3796 default_node_network_controller.go:1380] Starting node controller node sync W0423 20:43:20.114207 3796 egressip_healthcheck.go:76] Health checking using insecure connection I0423 20:43:20.114394 3796 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0423 20:43:20.125324 3796 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.096344ms I0423 20:43:20.125552 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-197.ec2.internal", IP: 10.0.1.197 to PMTUD blocking rules I0423 20:43:20.125568 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-197.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 20:43:20.131438 3796 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0423 20:43:20.131790 3796 egressservice_node.go:93] Setting up event handlers for Egress Services I0423 20:43:20.131977 3796 egressservice_node.go:183] Starting Egress Services Controller I0423 20:43:20.131996 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 20:43:20.132009 3796 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 20:43:20.132017 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 20:43:20.132024 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 20:43:20.132030 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 20:43:20.132035 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 20:43:20.132039 3796 egressservice_node.go:197] Repairing Egress Services I0423 20:43:20.142634 3796 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0423 20:43:20.142653 3796 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 20:43:20.142854 3796 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0423 20:43:20.142868 3796 link_network_manager.go:119] Link manager is running I0423 20:43:20.142881 3796 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0423 20:43:20.143087 3796 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0423 20:43:20.143562 3796 vrf_manager.go:128] VRF manager is running I0423 20:43:20.143677 3796 node_controller_manager.go:437] Removing flows to drop GARP I0423 20:43:20.143691 3796 gateway.go:532] Reconciling gateway with updates I0423 20:43:20.168318 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:20.168338 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:20.168354 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:20.168361 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:20.168383 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 20:43:20.168393 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:20.168403 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:20.168405 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 20:43:20.168410 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:20.168409 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 20:43:20.168416 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:20.168426 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 20:43:20.168428 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:20.168433 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 20:43:20.168435 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:20.168441 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:20.168451 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:20.168452 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:20.168352 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:20.168462 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:20.168464 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 20:43:20.168465 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:20.168388 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 20:43:20.168411 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:20.168468 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:20.168477 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:20.168479 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 20:43:20.168486 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 20:43:20.168488 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:20.168491 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:20.168494 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:20.168504 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 20:43:20.168507 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:20.168514 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 20:43:20.168515 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:20.168520 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:20.168517 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:20.168516 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:20.168528 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:20.168528 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:20.168531 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:20.168541 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:20.168477 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:20.168544 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:20.168546 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:20.168551 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 20:43:20.168396 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:20.168557 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:20.168563 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:20.168568 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:20.168572 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:20.168577 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 20:43:20.168477 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 20:43:20.168452 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:20.168559 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:20.168585 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 20:43:20.168586 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 20:43:20.168594 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:20.168593 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:20.168544 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:20.168596 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:20.168603 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:20.168605 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 20:43:20.168606 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:20.168610 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 20:43:20.168649 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:20.168593 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 32482 is already open, no action needed I0423 20:43:20.168706 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 32393 is already open, no action needed I0423 20:43:20.168500 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:20.168728 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:20.168437 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:20.168753 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:20.168770 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:20.168791 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:20.168567 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:20.168481 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:20.168852 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:20.168480 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:20.168877 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:20.168882 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 20:43:20.168460 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:20.168890 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:20.168491 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:20.168905 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:20.168501 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:20.168967 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 20:43:20.168498 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:20.168976 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:20.168502 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:20.169013 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:20.168515 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:20.169042 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:20.168525 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 20:43:20.168530 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:20.169095 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 20:43:20.168543 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:20.169103 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:20.169106 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 20:43:20.168550 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:20.169113 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:20.168577 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 20:43:20.168579 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:20.168586 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 20:43:20.168479 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:20.169163 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:20.168592 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:20.168599 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:20.169251 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:20.168526 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:20.210206 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 20:43:20.210240 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 20:43:20.210260 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 20:43:20.210273 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 20:43:20.210280 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 20:43:20.210289 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 20:43:20.210295 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 20:43:20.210303 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 20:43:20.250877 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 20:43:20.250887 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 20:43:20.250916 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 20:43:20.250923 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 20:43:20.250932 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 20:43:20.250937 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 20:43:20.250943 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 20:43:20.250948 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 20:43:20.250954 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 20:43:20.250961 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 20:43:20.250967 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 20:43:20.250972 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 20:43:20.250979 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 20:43:20.250984 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 20:43:20.250990 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 20:43:20.250996 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 20:43:20.289897 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 20:43:22.530431 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-197.ec2.internal I0423 20:43:22.530472 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-197.ec2.internal: 47.372µs I0423 20:43:22.546753 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-172.ec2.internal I0423 20:43:22.546775 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-172.ec2.internal: 28.007µs I0423 20:43:22.591472 3796 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6c5ccb6fb-2t4mj] creating logical port openshift-image-registry_image-registry-6c5ccb6fb-2t4mj for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:22.591594 3796 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29616280-2tvcw for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:22.591672 3796 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-mx9fr for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:22.591706 3796 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw I0423 20:43:22.591742 3796 kube.go:256] Updating pod openshift-image-registry/image-registry-6c5ccb6fb-2t4mj I0423 20:43:22.591747 3796 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr I0423 20:43:22.613688 3796 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr] pod update took 21.969624ms I0423 20:43:22.613703 3796 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr] addLogicalPort annotation time took 21.986598ms I0423 20:43:22.613808 3796 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw] pod update took 22.122576ms I0423 20:43:22.613824 3796 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw] addLogicalPort annotation time took 22.142925ms I0423 20:43:22.614908 3796 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr] addLogicalPort took 23.242261ms, libovsdb time 784.986µs I0423 20:43:22.615179 3796 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw] addLogicalPort took 23.601108ms, libovsdb time 914.095µs I0423 20:43:22.617827 3796 pod.go:62] [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj] pod update took 26.107124ms I0423 20:43:22.617988 3796 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6c5ccb6fb-2t4mj] addLogicalPort annotation time took 26.269328ms I0423 20:43:22.621427 3796 pods.go:271] [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj] addLogicalPort took 29.972157ms, libovsdb time 2.11632ms I0423 20:43:22.625430 3796 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-j6z2f] creating logical port openshift-dns_dns-default-j6z2f for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:22.625646 3796 kube.go:256] Updating pod openshift-dns/dns-default-j6z2f I0423 20:43:22.637973 3796 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-pqfrt] creating logical port openshift-ingress-canary_ingress-canary-pqfrt for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:22.638077 3796 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-pqfrt I0423 20:43:22.642142 3796 pod.go:62] [openshift-dns/dns-default-j6z2f] pod update took 16.543296ms I0423 20:43:22.642161 3796 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-j6z2f] addLogicalPort annotation time took 16.563914ms I0423 20:43:22.643016 3796 pods.go:271] [openshift-dns/dns-default-j6z2f] addLogicalPort took 17.597869ms, libovsdb time 646.963µs I0423 20:43:22.650967 3796 pod.go:62] [openshift-ingress-canary/ingress-canary-pqfrt] pod update took 12.921598ms I0423 20:43:22.650987 3796 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-pqfrt] addLogicalPort annotation time took 12.94571ms I0423 20:43:22.651924 3796 pods.go:271] [openshift-ingress-canary/ingress-canary-pqfrt] addLogicalPort took 13.963468ms, libovsdb time 672.481µs I0423 20:43:23.015696 3796 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key ] I0423 20:43:23.019485 3796 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr f407f2822bc826ef5f47ff987d1e75d4cacddd4e5f7a933e27f9df54cd115fd0 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr f407f2822bc826ef5f47ff987d1e75d4cacddd4e5f7a933e27f9df54cd115fd0 network default NAD default NAD key ] I0423 20:43:23.036461 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29616280-2tvcw, hostIfaceName: 88cd1c0ce2d1cd9, network: default, NAD default, SandboxID: "88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48", PCI device ID: , UID: "d3123090-5a15-43a7-a95f-28e2b5b8a591", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0423 20:43:23.036494 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-mx9fr, hostIfaceName: f407f2822bc826e, network: default, NAD default, SandboxID: "f407f2822bc826ef5f47ff987d1e75d4cacddd4e5f7a933e27f9df54cd115fd0", PCI device ID: , UID: "648f8f1f-94ba-462e-bea8-af59669d4a34", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0423 20:43:23.042292 3796 cni.go:452] [openshift-ingress-canary/ingress-canary-pqfrt 7618a2f3017fe35f77c2a0d68b88d101df08027f200786063d36a7e034f96951 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-pqfrt 7618a2f3017fe35f77c2a0d68b88d101df08027f200786063d36a7e034f96951 network default NAD default NAD key ] I0423 20:43:23.054655 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-pqfrt, hostIfaceName: 7618a2f3017fe35, network: default, NAD default, SandboxID: "7618a2f3017fe35f77c2a0d68b88d101df08027f200786063d36a7e034f96951", PCI device ID: , UID: "2a3d6139-2ef4-4c12-9341-bf41a40b6f8b", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0423 20:43:23.085459 3796 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:48:cf:4f:3a:0c\",\"name\":\"88cd1c0ce2d1cd9\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0984ebb3-aced-452c-96e9-ff6a74cc8095\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:23.092898 3796 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr f407f2822bc826ef5f47ff987d1e75d4cacddd4e5f7a933e27f9df54cd115fd0 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mx9fr f407f2822bc826ef5f47ff987d1e75d4cacddd4e5f7a933e27f9df54cd115fd0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:82:b4:0f:ee:48\",\"name\":\"f407f2822bc826e\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb87ed09-e9a1-4766-a6a1-c41260ef6ddc\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:23.102200 3796 cni.go:473] [openshift-ingress-canary/ingress-canary-pqfrt 7618a2f3017fe35f77c2a0d68b88d101df08027f200786063d36a7e034f96951 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-pqfrt 7618a2f3017fe35f77c2a0d68b88d101df08027f200786063d36a7e034f96951 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:5a:50:44:70:b7\",\"name\":\"7618a2f3017fe35\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9cec6832-6db0-4b22-8c80-5237f7739008\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:24.571703 3796 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-rq4fg] creating logical port openshift-insights_insights-operator-6dbb575d6c-rq4fg for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:24.571802 3796 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-rq4fg I0423 20:43:24.593950 3796 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-rq4fg] pod update took 22.174506ms I0423 20:43:24.594037 3796 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-rq4fg] addLogicalPort annotation time took 22.270514ms I0423 20:43:24.595506 3796 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-rq4fg] addLogicalPort took 23.816569ms, libovsdb time 1.190769ms I0423 20:43:24.604995 3796 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-67455d6b5-sw7sv] creating logical port openshift-ingress_router-default-67455d6b5-sw7sv for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:24.605078 3796 kube.go:256] Updating pod openshift-ingress/router-default-67455d6b5-sw7sv I0423 20:43:24.631233 3796 pod.go:62] [openshift-ingress/router-default-67455d6b5-sw7sv] pod update took 26.180766ms I0423 20:43:24.631257 3796 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-67455d6b5-sw7sv] addLogicalPort annotation time took 26.211166ms I0423 20:43:24.632458 3796 pods.go:271] [openshift-ingress/router-default-67455d6b5-sw7sv] addLogicalPort took 27.472591ms, libovsdb time 927.966µs I0423 20:43:24.682517 3796 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-zwxg2 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:24.682613 3796 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2 I0423 20:43:24.689559 3796 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-4qh22] creating logical port openshift-console-operator_console-operator-66dbbb6468-4qh22 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:24.689690 3796 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-4qh22 I0423 20:43:24.689732 3796 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-7c7tf for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:24.689806 3796 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf I0423 20:43:24.701685 3796 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2] pod update took 19.100655ms I0423 20:43:24.701712 3796 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2] addLogicalPort annotation time took 19.132133ms I0423 20:43:24.703036 3796 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2] addLogicalPort took 20.538026ms, libovsdb time 973.275µs I0423 20:43:24.718350 3796 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf] pod update took 28.557132ms I0423 20:43:24.718392 3796 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf] addLogicalPort annotation time took 28.587578ms I0423 20:43:24.718400 3796 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-4qh22] pod update took 28.737096ms I0423 20:43:24.718419 3796 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-4qh22] addLogicalPort annotation time took 28.759406ms I0423 20:43:24.719572 3796 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf] addLogicalPort took 29.850272ms, libovsdb time 833.659µs I0423 20:43:24.719955 3796 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-4qh22] addLogicalPort took 30.411041ms, libovsdb time 1.211116ms I0423 20:43:24.785918 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-7cb26 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:24.786040 3796 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26 I0423 20:43:24.809458 3796 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26] pod update took 23.440642ms I0423 20:43:24.809483 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26] addLogicalPort annotation time took 23.471326ms I0423 20:43:24.810677 3796 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26] addLogicalPort took 24.773583ms, libovsdb time 890.452µs I0423 20:43:24.998620 3796 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-rq4fg b862dcea7874c0ca0714cb2a940b5ada54e7a54fe199063f868d3860c1592878 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-rq4fg b862dcea7874c0ca0714cb2a940b5ada54e7a54fe199063f868d3860c1592878 network default NAD default NAD key ] I0423 20:43:25.010103 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-rq4fg, hostIfaceName: b862dcea7874c0c, network: default, NAD default, SandboxID: "b862dcea7874c0ca0714cb2a940b5ada54e7a54fe199063f868d3860c1592878", PCI device ID: , UID: "15a6e583-6b42-465e-a805-57bd4bc62fd6", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0423 20:43:25.046031 3796 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2 5f5a6037ea660b70931c882a4b79f68d0f2992e5766f40fb3027dab6f9a725d2 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2 5f5a6037ea660b70931c882a4b79f68d0f2992e5766f40fb3027dab6f9a725d2 network default NAD default NAD key ] I0423 20:43:25.058233 3796 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-4qh22 0c12e3c4b869fffbc68b7fc33477480810af6c95e0f115073dcd777fbe181658 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-4qh22 0c12e3c4b869fffbc68b7fc33477480810af6c95e0f115073dcd777fbe181658 network default NAD default NAD key ] I0423 20:43:25.059670 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-zwxg2, hostIfaceName: 5f5a6037ea660b7, network: default, NAD default, SandboxID: "5f5a6037ea660b70931c882a4b79f68d0f2992e5766f40fb3027dab6f9a725d2", PCI device ID: , UID: "3a8f9fff-90b0-4e76-8401-de3be7b37bbd", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0423 20:43:25.062869 3796 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-rq4fg b862dcea7874c0ca0714cb2a940b5ada54e7a54fe199063f868d3860c1592878 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-rq4fg b862dcea7874c0ca0714cb2a940b5ada54e7a54fe199063f868d3860c1592878 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:85:6e:82:c8:91\",\"name\":\"b862dcea7874c0c\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8c6af5c-8143-48dc-8fbe-547ab422096f\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:25.071282 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-4qh22, hostIfaceName: 0c12e3c4b869fff, network: default, NAD default, SandboxID: "0c12e3c4b869fffbc68b7fc33477480810af6c95e0f115073dcd777fbe181658", PCI device ID: , UID: "854ecea2-49b1-4a85-91bc-1d7d51acebc7", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0423 20:43:25.108982 3796 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2 5f5a6037ea660b70931c882a4b79f68d0f2992e5766f40fb3027dab6f9a725d2 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-zwxg2 5f5a6037ea660b70931c882a4b79f68d0f2992e5766f40fb3027dab6f9a725d2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:2e:a4:99:5b:fc\",\"name\":\"5f5a6037ea660b7\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0423d23d-19fd-4acc-9052-7f4e09d65143\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:25.120632 3796 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-4qh22 0c12e3c4b869fffbc68b7fc33477480810af6c95e0f115073dcd777fbe181658 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-4qh22 0c12e3c4b869fffbc68b7fc33477480810af6c95e0f115073dcd777fbe181658 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:0a:20:ed:7c:1b\",\"name\":\"0c12e3c4b869fff\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db509138-db03-4907-b231-9d02f09994ba\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:43:25.942813 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:26.750092 3796 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0423 20:43:26.750145 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:26.750160 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 21.935µs I0423 20:43:26.750181 3796 default_network_controller.go:1025] Node "ip-10-0-2-22.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:29.781169 3796 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0423 20:43:29.782117 3796 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 926.648µs I0423 20:43:29.854364 3796 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0423 20:43:29.982011 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 20:43:30.882839 3796 namespace.go:98] [openshift-service-ca] adding namespace I0423 20:43:30.883682 3796 namespace.go:102] [openshift-service-ca] adding namespace took 825.089µs I0423 20:43:30.904819 3796 namespace.go:142] [openshift-service-ca] updating namespace W0423 20:43:30.984189 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 20:43:31.958928 3796 default_network_controller.go:1025] Node "ip-10-0-2-22.ec2.internal" in remote zone "ip-10-0-2-22.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:31.958977 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0423 20:43:31.959013 3796 node_tracker.go:235] Node ip-10-0-2-22.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-22.ec2.internal" I0423 20:43:31.959028 3796 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:31.959042 3796 services_controller.go:574] Full service sync requested for network=default W0423 20:43:31.959362 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 20:43:31.959552 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:43:31.959629 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 20:43:32.113589 3796 default_network_controller.go:1025] Node "ip-10-0-2-22.ec2.internal" in remote zone "ip-10-0-2-22.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:32.113596 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0423 20:43:32.113679 3796 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:32.113694 3796 services_controller.go:574] Full service sync requested for network=default W0423 20:43:32.114027 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 20:43:32.114220 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:43:32.114229 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 20:43:32.114735 3796 zone_ic_handler.go:195] Time taken to create transit switch: 286.621µs I0423 20:43:32.114749 3796 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-22.ec2.internal for the network default I0423 20:43:32.115725 3796 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-22.ec2.internal" on network "default" took: 1.288768ms I0423 20:43:32.563925 3796 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-94b6h] creating logical port openshift-service-ca_service-ca-d9cb48497-94b6h for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:32.564027 3796 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-94b6h I0423 20:43:32.581309 3796 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-94b6h] pod update took 17.324441ms I0423 20:43:32.581392 3796 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-94b6h] addLogicalPort annotation time took 17.353606ms I0423 20:43:32.582429 3796 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-94b6h] addLogicalPort took 18.521443ms, libovsdb time 744.382µs I0423 20:43:32.921333 3796 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-94b6h 161cbd63e7c7fd91347173928a969c6d3b98d0c9cd06352a9299a002779c7e3a network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-94b6h 161cbd63e7c7fd91347173928a969c6d3b98d0c9cd06352a9299a002779c7e3a network default NAD default NAD key ] I0423 20:43:32.931276 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-94b6h, hostIfaceName: 161cbd63e7c7fd9, network: default, NAD default, SandboxID: "161cbd63e7c7fd91347173928a969c6d3b98d0c9cd06352a9299a002779c7e3a", PCI device ID: , UID: "ad2d1664-3184-444a-ab20-fa59cbf8d9ae", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0423 20:43:32.978287 3796 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-94b6h 161cbd63e7c7fd91347173928a969c6d3b98d0c9cd06352a9299a002779c7e3a network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-94b6h 161cbd63e7c7fd91347173928a969c6d3b98d0c9cd06352a9299a002779c7e3a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:fb:ed:44:21:8d\",\"name\":\"161cbd63e7c7fd9\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42ef8631-9609-467d-97c8-91283a4c4c96\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:43:33.914610 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:34.674673 3796 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0423 20:43:34.674685 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:34.674706 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 26.538µs I0423 20:43:34.674728 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-22.ec2.internal", IP: 10.0.2.22 to PMTUD blocking rules I0423 20:43:34.674731 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0423 20:43:34.674739 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-22.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 20:43:34.674781 3796 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:34.674797 3796 services_controller.go:574] Full service sync requested for network=default W0423 20:43:34.674881 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 20:43:34.675234 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 20:43:34.680840 3796 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-22.ec2.internal. Updating no re-route policies I0423 20:43:34.689270 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-22.ec2.internal", IP: 10.0.2.22 to PMTUD blocking rules I0423 20:43:34.689286 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-22.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0423 20:43:37.271732 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6d44d5bbf5-f4bml IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6d44d5bbf5-f4bml: no pod IPs found W0423 20:43:37.670052 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 20:43:38.619202 3796 cni.go:452] [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key ] I0423 20:43:38.629115 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:38.629147 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 38.935µs I0423 20:43:38.630694 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6c5ccb6fb-2t4mj, hostIfaceName: ddb4718c045df2d, network: default, NAD default, SandboxID: "ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2", PCI device ID: , UID: "a32a60f1-d30d-4dfe-b217-b6e3d60eb269", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0423 20:43:38.653195 3796 cni.go:452] [openshift-dns/dns-default-j6z2f f89cbc281e63704725216cf078e75b96ac69a7e7d1a73d6d8c7e3ba831f3b484 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-j6z2f f89cbc281e63704725216cf078e75b96ac69a7e7d1a73d6d8c7e3ba831f3b484 network default NAD default NAD key ] I0423 20:43:38.664388 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-j6z2f, hostIfaceName: f89cbc281e63704, network: default, NAD default, SandboxID: "f89cbc281e63704725216cf078e75b96ac69a7e7d1a73d6d8c7e3ba831f3b484", PCI device ID: , UID: "0d9294f3-6b3a-4f4c-aa8a-d007b973b2f0", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0423 20:43:38.681931 3796 cni.go:473] [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:aa:d4:5d:83:97\",\"name\":\"ddb4718c045df2d\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6883584a-8ad4-47a5-8e2c-2b434ff6bffb\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:38.710438 3796 cni.go:473] [openshift-dns/dns-default-j6z2f f89cbc281e63704725216cf078e75b96ac69a7e7d1a73d6d8c7e3ba831f3b484 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-j6z2f f89cbc281e63704725216cf078e75b96ac69a7e7d1a73d6d8c7e3ba831f3b484 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:6a:17:2b:48:9b\",\"name\":\"f89cbc281e63704\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18416022-282f-497e-8d7c-1d16dd92b35d\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:43:38.914935 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 20:43:38.970871 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:43:38.971521 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:43:38.986196 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:43:38.986214 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:43:38.994986 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c5ccb6fb-bgb65 I0423 20:43:38.995009 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c5ccb6fb-bgb65 I0423 20:43:50.756599 3796 cni.go:452] [openshift-multus/network-metrics-daemon-47kfb 190c32b073236b37cd3a10144694da1a9e0df712957d60a979559f63e9967452 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-47kfb 190c32b073236b37cd3a10144694da1a9e0df712957d60a979559f63e9967452 network default NAD default NAD key ] I0423 20:43:50.766772 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-47kfb, hostIfaceName: 190c32b073236b3, network: default, NAD default, SandboxID: "190c32b073236b37cd3a10144694da1a9e0df712957d60a979559f63e9967452", PCI device ID: , UID: "f6409fab-428d-49a7-b7d3-8692a9fa6065", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0423 20:43:50.779862 3796 cni.go:452] [openshift-network-diagnostics/network-check-target-94wvc 1eeeb23023d49a8bebc1125a34e53719ad0da849f34c82443d79611342c1b599 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-94wvc 1eeeb23023d49a8bebc1125a34e53719ad0da849f34c82443d79611342c1b599 network default NAD default NAD key ] I0423 20:43:50.789466 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-94wvc, hostIfaceName: 1eeeb23023d49a8, network: default, NAD default, SandboxID: "1eeeb23023d49a8bebc1125a34e53719ad0da849f34c82443d79611342c1b599", PCI device ID: , UID: "337392bc-b34e-4636-8c5e-67fca4205477", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0423 20:43:50.818200 3796 cni.go:473] [openshift-multus/network-metrics-daemon-47kfb 190c32b073236b37cd3a10144694da1a9e0df712957d60a979559f63e9967452 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-47kfb 190c32b073236b37cd3a10144694da1a9e0df712957d60a979559f63e9967452 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:5f:4c:7f:7a:ac\",\"name\":\"190c32b073236b3\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/880ece62-d9f4-4e43-afc2-2e459088fff7\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:50.836446 3796 cni.go:473] [openshift-network-diagnostics/network-check-target-94wvc 1eeeb23023d49a8bebc1125a34e53719ad0da849f34c82443d79611342c1b599 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-94wvc 1eeeb23023d49a8bebc1125a34e53719ad0da849f34c82443d79611342c1b599 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:94:f0:56:0b:9d\",\"name\":\"1eeeb23023d49a8\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fcc478de-8abb-469a-91a9-d71ef06fa426\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:43:55.443927 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0423 20:43:55.456696 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:43:55.459259 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:43:55.592668 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 20:43:55.595845 3796 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-fzj9m] creating logical port openshift-insights_insights-runtime-extractor-fzj9m for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:55.595933 3796 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-fzj9m I0423 20:43:55.614794 3796 pod.go:62] [openshift-insights/insights-runtime-extractor-fzj9m] pod update took 18.882856ms I0423 20:43:55.614815 3796 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-fzj9m] addLogicalPort annotation time took 18.909851ms I0423 20:43:55.615955 3796 pods.go:271] [openshift-insights/insights-runtime-extractor-fzj9m] addLogicalPort took 20.124064ms, libovsdb time 824.847µs W0423 20:43:55.642472 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 20:43:55.937405 3796 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-78trm] creating logical port openshift-console_downloads-5c54f4944f-78trm for pod on switch ip-10-0-0-172.ec2.internal I0423 20:43:55.937482 3796 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-78trm I0423 20:43:55.967728 3796 pod.go:62] [openshift-console/downloads-5c54f4944f-78trm] pod update took 30.265682ms I0423 20:43:55.967755 3796 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-78trm] addLogicalPort annotation time took 30.295557ms I0423 20:43:55.968758 3796 pods.go:271] [openshift-console/downloads-5c54f4944f-78trm] addLogicalPort took 31.38636ms, libovsdb time 735.82µs I0423 20:43:56.254637 3796 cni.go:452] [openshift-insights/insights-runtime-extractor-fzj9m da735e5695a271ea37ba82f026148ea4af8443ac4d5e6f8dfb0fffc259ad24b5 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-fzj9m da735e5695a271ea37ba82f026148ea4af8443ac4d5e6f8dfb0fffc259ad24b5 network default NAD default NAD key ] I0423 20:43:56.264028 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-fzj9m, hostIfaceName: da735e5695a271e, network: default, NAD default, SandboxID: "da735e5695a271ea37ba82f026148ea4af8443ac4d5e6f8dfb0fffc259ad24b5", PCI device ID: , UID: "eadf433f-4da1-4ebc-a166-41bfb1f21155", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0423 20:43:56.314010 3796 cni.go:473] [openshift-insights/insights-runtime-extractor-fzj9m da735e5695a271ea37ba82f026148ea4af8443ac4d5e6f8dfb0fffc259ad24b5 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-fzj9m da735e5695a271ea37ba82f026148ea4af8443ac4d5e6f8dfb0fffc259ad24b5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:4c:de:37:6a:97\",\"name\":\"da735e5695a271e\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b74734c-2286-4106-bd21-4a6a3263e523\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:56.339520 3796 cni.go:452] [openshift-console/downloads-5c54f4944f-78trm f6815619ad201dfde2363969bf5ac1bae2d33416976f2db6adf97f7e5fa9ec69 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-78trm f6815619ad201dfde2363969bf5ac1bae2d33416976f2db6adf97f7e5fa9ec69 network default NAD default NAD key ] I0423 20:43:56.349563 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-78trm, hostIfaceName: f6815619ad201df, network: default, NAD default, SandboxID: "f6815619ad201dfde2363969bf5ac1bae2d33416976f2db6adf97f7e5fa9ec69", PCI device ID: , UID: "ee0a8cc6-3924-4fd9-8323-3fbc36d04488", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0423 20:43:56.397972 3796 cni.go:473] [openshift-console/downloads-5c54f4944f-78trm f6815619ad201dfde2363969bf5ac1bae2d33416976f2db6adf97f7e5fa9ec69 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-78trm f6815619ad201dfde2363969bf5ac1bae2d33416976f2db6adf97f7e5fa9ec69 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:c1:8d:64:92:98\",\"name\":\"f6815619ad201df\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e6c49b39-26f6-40b9-9e58-46896e0e0428\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:56.778767 3796 cni.go:452] [openshift-ingress/router-default-67455d6b5-sw7sv a5d6b40499c5168ebb533b7de6fd1a24d8915c7cb5607ef2d5ecb48180a59cd4 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-67455d6b5-sw7sv a5d6b40499c5168ebb533b7de6fd1a24d8915c7cb5607ef2d5ecb48180a59cd4 network default NAD default NAD key ] I0423 20:43:56.788604 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-67455d6b5-sw7sv, hostIfaceName: a5d6b40499c5168, network: default, NAD default, SandboxID: "a5d6b40499c5168ebb533b7de6fd1a24d8915c7cb5607ef2d5ecb48180a59cd4", PCI device ID: , UID: "6a3e1f37-d590-4c2e-a57c-4c1126e260f1", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0423 20:43:56.838067 3796 cni.go:473] [openshift-ingress/router-default-67455d6b5-sw7sv a5d6b40499c5168ebb533b7de6fd1a24d8915c7cb5607ef2d5ecb48180a59cd4 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-67455d6b5-sw7sv a5d6b40499c5168ebb533b7de6fd1a24d8915c7cb5607ef2d5ecb48180a59cd4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:7e:b8:d8:7f:53\",\"name\":\"a5d6b40499c5168\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6fd7e863-e5c5-4487-bf93-169b664f0c6c\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:56.870030 3796 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf c57cfcd02aa3d94af4e0d5882483f2a883e24a419449d760c19d1aab7450a1a2 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf c57cfcd02aa3d94af4e0d5882483f2a883e24a419449d760c19d1aab7450a1a2 network default NAD default NAD key ] I0423 20:43:56.882649 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-7c7tf, hostIfaceName: c57cfcd02aa3d94, network: default, NAD default, SandboxID: "c57cfcd02aa3d94af4e0d5882483f2a883e24a419449d760c19d1aab7450a1a2", PCI device ID: , UID: "0ca36b7d-6590-4736-ad80-462fe9a01691", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0423 20:43:56.935639 3796 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf c57cfcd02aa3d94af4e0d5882483f2a883e24a419449d760c19d1aab7450a1a2 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-7c7tf c57cfcd02aa3d94af4e0d5882483f2a883e24a419449d760c19d1aab7450a1a2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:e5:36:da:fe:dc\",\"name\":\"c57cfcd02aa3d94\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/824b5299-31e9-40d6-8cfe-9d9bdb090a76\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:43:56.968721 3796 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26 6706facc883e14ce7e175181a479abd393e614f3908beff77beb17b39255b8e1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26 6706facc883e14ce7e175181a479abd393e614f3908beff77beb17b39255b8e1 network default NAD default NAD key ] I0423 20:43:56.980263 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-7cb26, hostIfaceName: 6706facc883e14c, network: default, NAD default, SandboxID: "6706facc883e14ce7e175181a479abd393e614f3908beff77beb17b39255b8e1", PCI device ID: , UID: "77da227a-c39c-4eae-8913-4e44de9daa88", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] W0423 20:43:56.993537 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:56.993565 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:56.993568 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:56.993582 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:56.993588 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0423 20:43:57.032815 3796 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26 6706facc883e14ce7e175181a479abd393e614f3908beff77beb17b39255b8e1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-7cb26 6706facc883e14ce7e175181a479abd393e614f3908beff77beb17b39255b8e1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:ff:37:d5:93:42\",\"name\":\"6706facc883e14c\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01e72b17-7054-4629-b74e-5307d974f598\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:43:57.039844 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.039869 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.043798 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.043821 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.043829 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.075673 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.075689 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.076486 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.076502 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.076510 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.998344 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.998402 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default I0423 20:43:58.071578 3796 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0423 20:43:58.089149 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:58.089167 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:58.089173 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:44:00.992799 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:00.992821 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:01.952329 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:01.952353 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:03.005061 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0423 20:44:03.205354 3796 base_network_controller_pods.go:487] [default/openshift-console/console-7b549f9c9d-cmgt9] creating logical port openshift-console_console-7b549f9c9d-cmgt9 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:44:03.205483 3796 kube.go:256] Updating pod openshift-console/console-7b549f9c9d-cmgt9 I0423 20:44:03.227466 3796 pod.go:62] [openshift-console/console-7b549f9c9d-cmgt9] pod update took 22.01579ms I0423 20:44:03.227495 3796 base_network_controller_pods.go:951] [default/openshift-console/console-7b549f9c9d-cmgt9] addLogicalPort annotation time took 22.051133ms I0423 20:44:03.228784 3796 pods.go:271] [openshift-console/console-7b549f9c9d-cmgt9] addLogicalPort took 23.446769ms, libovsdb time 857.908µs I0423 20:44:03.570426 3796 cni.go:452] [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key ] I0423 20:44:03.581628 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7b549f9c9d-cmgt9, hostIfaceName: cf240b18fe3dac2, network: default, NAD default, SandboxID: "cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e", PCI device ID: , UID: "33cfbd1e-60e6-477a-a913-324480e88cee", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0423 20:44:03.635344 3796 cni.go:473] [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:1e:f0:73:db:e2\",\"name\":\"cf240b18fe3dac2\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eebe338f-ea0b-4245-8fa4-ba004f06eb66\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:44:07.039467 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 20:44:08.055239 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 20:44:08.097057 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 20:44:14.084464 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:14.175654 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:14.175675 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:14.175682 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 20:44:15.251809 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-p5s49 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:44:15.251928 3796 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49 I0423 20:44:15.274318 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-5b4968d44-jgp6v] creating logical port openshift-monitoring_metrics-server-5b4968d44-jgp6v for pod on switch ip-10-0-0-172.ec2.internal I0423 20:44:15.274463 3796 kube.go:256] Updating pod openshift-monitoring/metrics-server-5b4968d44-jgp6v I0423 20:44:15.277896 3796 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49] pod update took 26.001266ms I0423 20:44:15.277920 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49] addLogicalPort annotation time took 26.025801ms I0423 20:44:15.279425 3796 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49] addLogicalPort took 27.631602ms, libovsdb time 969.766µs I0423 20:44:15.298029 3796 pod.go:62] [openshift-monitoring/metrics-server-5b4968d44-jgp6v] pod update took 23.611967ms I0423 20:44:15.298050 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-5b4968d44-jgp6v] addLogicalPort annotation time took 23.636127ms I0423 20:44:15.299236 3796 pods.go:271] [openshift-monitoring/metrics-server-5b4968d44-jgp6v] addLogicalPort took 24.939508ms, libovsdb time 814.83µs I0423 20:44:16.480671 3796 cni.go:452] [openshift-monitoring/metrics-server-5b4968d44-jgp6v 78e7ff01e706bd34254f1315beadf94649446547affc33a42f9f8803333c2e74 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-5b4968d44-jgp6v 78e7ff01e706bd34254f1315beadf94649446547affc33a42f9f8803333c2e74 network default NAD default NAD key ] I0423 20:44:16.481140 3796 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49 bd018bfda422e05804dd45d4139305aea6b1cfdcd1bc8529863c079677efb6a0 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49 bd018bfda422e05804dd45d4139305aea6b1cfdcd1bc8529863c079677efb6a0 network default NAD default NAD key ] I0423 20:44:16.506117 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-5b4968d44-jgp6v, hostIfaceName: 78e7ff01e706bd3, network: default, NAD default, SandboxID: "78e7ff01e706bd34254f1315beadf94649446547affc33a42f9f8803333c2e74", PCI device ID: , UID: "227c1952-3fa3-4077-8f7b-37b2d92cfa30", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0423 20:44:16.509301 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-p5s49, hostIfaceName: bd018bfda422e05, network: default, NAD default, SandboxID: "bd018bfda422e05804dd45d4139305aea6b1cfdcd1bc8529863c079677efb6a0", PCI device ID: , UID: "4e8e3918-c507-4875-a884-d8f2becbf3ef", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0423 20:44:16.558828 3796 cni.go:473] [openshift-monitoring/metrics-server-5b4968d44-jgp6v 78e7ff01e706bd34254f1315beadf94649446547affc33a42f9f8803333c2e74 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-5b4968d44-jgp6v 78e7ff01e706bd34254f1315beadf94649446547affc33a42f9f8803333c2e74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:54:d3:d4:b9:35\",\"name\":\"78e7ff01e706bd3\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a506b10d-0dd5-4456-a992-9b0c633ee4c9\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:44:16.580622 3796 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49 bd018bfda422e05804dd45d4139305aea6b1cfdcd1bc8529863c079677efb6a0 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-p5s49 bd018bfda422e05804dd45d4139305aea6b1cfdcd1bc8529863c079677efb6a0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:a6:5e:25:42:3e\",\"name\":\"bd018bfda422e05\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd0169bd-01d5-41dd-875e-d3789e768d34\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:44:16.969579 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:16.969620 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:17.005980 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-7cc8fc68d5-d94fz IP address from the namespace address-set, err: pod openshift-console/console-7cc8fc68d5-d94fz: no pod IPs found W0423 20:44:17.087583 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0423 20:44:17.102078 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0423 20:44:17.127191 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0423 20:44:18.218435 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 20:44:19.096745 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0423 20:44:19.141759 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0423 20:44:19.174482 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 20:44:19.197858 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:44:19.197907 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:44:19.229328 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:19.229352 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:19.229359 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:19.229365 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:20.350768 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:20.350792 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:20.350800 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:20.527606 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:44:20.527636 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0423 20:44:20.738739 3796 cni.go:452] [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key ] I0423 20:44:20.812816 3796 cni.go:473] [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-6c5ccb6fb-2t4mj ddb4718c045df2dd59d56f07b48d622c07e9df7f14f1390323d78cc122040fb2 network default NAD default NAD key default], result "{}", err I0423 20:44:20.871258 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.871300 3796 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6c5ccb6fb-2t4mj I0423 20:44:20.872699 3796 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6c5ccb6fb-2t4mj, ips: 10.128.0.6 I0423 20:44:20.872732 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.886970 3796 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-66bc67cd8c-6qlw6] creating logical port openshift-image-registry_image-registry-66bc67cd8c-6qlw6 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:44:20.887077 3796 kube.go:256] Updating pod openshift-image-registry/image-registry-66bc67cd8c-6qlw6 I0423 20:44:20.908450 3796 pod.go:62] [openshift-image-registry/image-registry-66bc67cd8c-6qlw6] pod update took 21.403149ms I0423 20:44:20.908478 3796 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-66bc67cd8c-6qlw6] addLogicalPort annotation time took 21.434083ms I0423 20:44:20.909728 3796 pods.go:271] [openshift-image-registry/image-registry-66bc67cd8c-6qlw6] addLogicalPort took 22.775849ms, libovsdb time 907.839µs I0423 20:44:20.986439 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.986963 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:21.119946 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.119970 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.126952 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c5ccb6fb-2t4mj I0423 20:44:21.126980 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c5ccb6fb-2t4mj I0423 20:44:21.180535 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.180560 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.190011 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-587b6c485f-ng4zk I0423 20:44:21.190042 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-587b6c485f-ng4zk I0423 20:44:21.249889 3796 cni.go:452] [openshift-image-registry/image-registry-66bc67cd8c-6qlw6 c48b7b76f8f683b04a339e5f07a6b370c268989cf85163968b55cfdbb0424765 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-66bc67cd8c-6qlw6 c48b7b76f8f683b04a339e5f07a6b370c268989cf85163968b55cfdbb0424765 network default NAD default NAD key ] I0423 20:44:21.260470 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-66bc67cd8c-6qlw6, hostIfaceName: c48b7b76f8f683b, network: default, NAD default, SandboxID: "c48b7b76f8f683b04a339e5f07a6b370c268989cf85163968b55cfdbb0424765", PCI device ID: , UID: "576bf97f-172f-4aab-92bc-fd93d076ae7f", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0423 20:44:21.313864 3796 cni.go:473] [openshift-image-registry/image-registry-66bc67cd8c-6qlw6 c48b7b76f8f683b04a339e5f07a6b370c268989cf85163968b55cfdbb0424765 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-66bc67cd8c-6qlw6 c48b7b76f8f683b04a339e5f07a6b370c268989cf85163968b55cfdbb0424765 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:4c:26:a0:d9:67\",\"name\":\"c48b7b76f8f683b\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4a0d1b7-9562-4825-9150-01cdcd4f8360\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:44:25.980557 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 20:44:33.332529 3796 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:33.333086 3796 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:34.227994 3796 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:34.228025 3796 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:34.235153 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b549f9c9d-7l4cm I0423 20:44:34.235181 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b549f9c9d-7l4cm W0423 20:44:39.187950 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 20:44:39.187970 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 20:44:40.117240 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0423 20:44:41.222480 3796 cni.go:452] [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key ] I0423 20:44:41.303027 3796 cni.go:473] [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7b549f9c9d-cmgt9 cf240b18fe3dac2f159c31d9c674090af81520c914e89e089a30f99ead93db3e network default NAD default NAD key default], result "{}", err I0423 20:44:41.377968 3796 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:41.378031 3796 pods.go:176] Deleting pod: openshift-console/console-7b549f9c9d-cmgt9 I0423 20:44:41.379297 3796 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7b549f9c9d-cmgt9, ips: 10.128.0.19 I0423 20:44:41.379323 3796 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:41.386448 3796 base_network_controller_pods.go:487] [default/openshift-console/console-665c78db9-wswd9] creating logical port openshift-console_console-665c78db9-wswd9 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:44:41.386535 3796 kube.go:256] Updating pod openshift-console/console-665c78db9-wswd9 I0423 20:44:41.401317 3796 pod.go:62] [openshift-console/console-665c78db9-wswd9] pod update took 14.809475ms I0423 20:44:41.401338 3796 base_network_controller_pods.go:951] [default/openshift-console/console-665c78db9-wswd9] addLogicalPort annotation time took 14.835ms I0423 20:44:41.402600 3796 pods.go:271] [openshift-console/console-665c78db9-wswd9] addLogicalPort took 16.167042ms, libovsdb time 824.446µs I0423 20:44:41.799669 3796 cni.go:452] [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key ] I0423 20:44:41.809361 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-665c78db9-wswd9, hostIfaceName: af1e4b41da88d4b, network: default, NAD default, SandboxID: "af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4", PCI device ID: , UID: "c95ddd93-c12c-48dd-b7b1-fe01ed2ed34e", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0423 20:44:41.859811 3796 cni.go:473] [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:01:77:a7:1b:dc\",\"name\":\"af1e4b41da88d4b\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/517e66dc-8544-43fe-a40e-716f5d9311ac\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:44:42.205735 3796 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:42.205756 3796 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:42.208678 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b549f9c9d-cmgt9 I0423 20:44:42.208706 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b549f9c9d-cmgt9 W0423 20:44:43.131846 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:44.257715 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:44.257738 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:44.257755 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:44.257785 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:44.287043 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0423 20:44:44.303481 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:44.303509 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:45.207217 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 20:44:45.207264 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 20:44:45.207221 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 20:44:46.209236 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0423 20:44:52.244235 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 20:44:59.265601 3796 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key ] I0423 20:44:59.344716 3796 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw 88cd1c0ce2d1cd9c309309fb30443a3440934b746fe5fe93e678d3aee6ddfb48 network default NAD default NAD key default], result "{}", err I0423 20:44:59.415674 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:59.415713 3796 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw I0423 20:44:59.417044 3796 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw, ips: 10.128.0.5 I0423 20:44:59.417070 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0423 20:45:00.272515 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0423 20:45:00.272573 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0423 20:45:00.272519 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0423 20:45:00.439424 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:00.439448 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0423 20:45:01.279174 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0423 20:45:09.471827 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:09.472400 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:09.609354 3796 obj_retry.go:498] Detected object openshift-console/console-7cc8fc68d5-s269m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:09.609908 3796 obj_retry.go:498] Detected object openshift-console/console-7cc8fc68d5-s269m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:10.405893 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.405926 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.412472 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-587b6c485f-rvc8z I0423 20:45:10.412492 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-587b6c485f-rvc8z I0423 20:45:10.430948 3796 obj_retry.go:492] Detected object openshift-console/console-7cc8fc68d5-s269m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.430967 3796 obj_retry.go:492] Detected object openshift-console/console-7cc8fc68d5-s269m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.437504 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7cc8fc68d5-s269m I0423 20:45:10.437521 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7cc8fc68d5-s269m I0423 20:45:13.338551 3796 obj_retry.go:498] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:13.339078 3796 obj_retry.go:498] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:13.717830 3796 obj_retry.go:492] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:13.717856 3796 obj_retry.go:492] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:13.724694 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7bcc69756c-9bl9j I0423 20:45:13.724708 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7bcc69756c-9bl9j I0423 20:45:32.717104 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:32.717629 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:33.511102 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:33.511123 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:33.518870 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0423 20:45:33.518896 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0423 20:45:37.126484 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:37.129147 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:37.552693 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:37.552719 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:37.559935 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0423 20:45:37.559963 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0423 20:45:49.129832 3796 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 for pod on switch ip-10-0-0-172.ec2.internal I0423 20:45:49.129965 3796 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 I0423 20:45:49.142411 3796 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6] pod update took 12.471515ms I0423 20:45:49.142440 3796 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6] addLogicalPort annotation time took 12.503007ms I0423 20:45:49.143904 3796 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6] addLogicalPort took 14.0914ms, libovsdb time 1.058618ms I0423 20:45:49.486576 3796 cni.go:452] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key ] I0423 20:45:49.495907 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6, hostIfaceName: 0cb37c5912efaf6, network: default, NAD default, SandboxID: "0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff", PCI device ID: , UID: "e4e292b5-8c1e-4c02-a5fd-ba62b3c41883", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0423 20:45:49.545529 3796 cni.go:473] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:f7:7b:e9:d0:cc\",\"name\":\"0cb37c5912efaf6\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3d730f1a-740f-40e8-b28b-06541fa3b576\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:46:05.213553 3796 obj_retry.go:498] Detected object openshift-console/console-665c78db9-pc9gx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.214149 3796 obj_retry.go:498] Detected object openshift-console/console-665c78db9-pc9gx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.324164 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.324648 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.590590 3796 obj_retry.go:492] Detected object openshift-console/console-665c78db9-pc9gx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.590618 3796 obj_retry.go:492] Detected object openshift-console/console-665c78db9-pc9gx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.597814 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-665c78db9-pc9gx I0423 20:46:05.597836 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-665c78db9-pc9gx I0423 20:46:05.620089 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.620112 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.627148 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0423 20:46:05.627176 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0423 20:46:10.543515 3796 cni.go:452] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key ] I0423 20:46:10.620653 3796 cni.go:473] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 0cb37c5912efaf662032651a6d3485f00bf2ae4a7017de74e5e906419e3c5dff network default NAD default NAD key default], result "{}", err I0423 20:46:10.668914 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.668949 3796 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 I0423 20:46:10.670336 3796 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6, ips: 10.128.0.24 I0423 20:46:10.670364 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.670402 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.670412 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0423 20:46:10.670645 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.670662 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0423 20:46:11.695821 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:11.695846 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:11.695863 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:11.695877 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:14.231450 3796 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 20:46:14.244447 3796 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 20:46:15.837893 3796 cni.go:452] [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key ] I0423 20:46:15.917923 3796 cni.go:473] [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-665c78db9-wswd9 af1e4b41da88d4b5a0e5adc2cd5f08c6402a76abf476ffc94f46e29a407cc8c4 network default NAD default NAD key default], result "{}", err I0423 20:46:15.968050 3796 obj_retry.go:498] Detected object openshift-console/console-665c78db9-wswd9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:15.968092 3796 pods.go:176] Deleting pod: openshift-console/console-665c78db9-wswd9 I0423 20:46:15.969307 3796 pods.go:236] Attempting to release IPs for pod: openshift-console/console-665c78db9-wswd9, ips: 10.128.0.23 I0423 20:46:15.969340 3796 obj_retry.go:498] Detected object openshift-console/console-665c78db9-wswd9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:16.565640 3796 obj_retry.go:492] Detected object openshift-console/console-665c78db9-wswd9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:16.565666 3796 obj_retry.go:492] Detected object openshift-console/console-665c78db9-wswd9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:16.572770 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-665c78db9-wswd9 I0423 20:46:16.572791 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-665c78db9-wswd9 I0423 20:46:40.204984 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:40.205560 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:40.744894 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:40.744918 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:40.752658 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0423 20:46:40.752681 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0423 20:47:48.067656 3796 base_network_controller_pods.go:487] [default/openshift-console/console-66899c5bfc-m4kpc] creating logical port openshift-console_console-66899c5bfc-m4kpc for pod on switch ip-10-0-0-172.ec2.internal I0423 20:47:48.067805 3796 kube.go:256] Updating pod openshift-console/console-66899c5bfc-m4kpc I0423 20:47:48.092487 3796 pod.go:62] [openshift-console/console-66899c5bfc-m4kpc] pod update took 24.717786ms I0423 20:47:48.092510 3796 base_network_controller_pods.go:951] [default/openshift-console/console-66899c5bfc-m4kpc] addLogicalPort annotation time took 24.746444ms I0423 20:47:48.093668 3796 pods.go:271] [openshift-console/console-66899c5bfc-m4kpc] addLogicalPort took 26.026633ms, libovsdb time 842.616µs I0423 20:47:48.428831 3796 cni.go:452] [openshift-console/console-66899c5bfc-m4kpc bb67415f2da2f321bca169ab31ed9e0c22297b1b397b6627aaf36846b48fcae1 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-66899c5bfc-m4kpc bb67415f2da2f321bca169ab31ed9e0c22297b1b397b6627aaf36846b48fcae1 network default NAD default NAD key ] I0423 20:47:48.438517 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-66899c5bfc-m4kpc, hostIfaceName: bb67415f2da2f32, network: default, NAD default, SandboxID: "bb67415f2da2f321bca169ab31ed9e0c22297b1b397b6627aaf36846b48fcae1", PCI device ID: , UID: "2e271806-da20-4097-a0a3-a3eeafb8972e", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0423 20:47:48.488580 3796 cni.go:473] [openshift-console/console-66899c5bfc-m4kpc bb67415f2da2f321bca169ab31ed9e0c22297b1b397b6627aaf36846b48fcae1 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-66899c5bfc-m4kpc bb67415f2da2f321bca169ab31ed9e0c22297b1b397b6627aaf36846b48fcae1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:b1:4c:97:fc:08\",\"name\":\"bb67415f2da2f32\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/255f1b1e-512d-46dc-bcae-d6b5c8a1b6cb\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:47:52.907167 3796 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0423 20:47:58.956693 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 20:48:13.369051 3796 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-sbb9l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:13.369621 3796 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-sbb9l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:14.134872 3796 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-sbb9l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:14.134898 3796 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-sbb9l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:14.143967 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86f657c86d-sbb9l I0423 20:48:14.143989 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86f657c86d-sbb9l I0423 20:48:19.909124 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0423 20:48:24.296882 3796 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-mhknr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:24.297474 3796 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-mhknr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:25.255053 3796 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-mhknr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:25.255076 3796 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-mhknr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:25.262753 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86f657c86d-mhknr I0423 20:48:25.262776 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86f657c86d-mhknr I0423 20:48:26.915319 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0423 20:48:57.814487 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=146 I0423 20:49:19.927394 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0423 20:49:22.904316 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0423 20:49:23.812895 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0423 20:50:00.188481 3796 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29616290-pfkpr for pod on switch ip-10-0-0-172.ec2.internal I0423 20:50:00.188623 3796 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr I0423 20:50:00.209742 3796 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr] pod update took 21.152798ms I0423 20:50:00.209765 3796 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr] addLogicalPort annotation time took 21.181275ms I0423 20:50:00.210956 3796 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr] addLogicalPort took 22.49541ms, libovsdb time 867.019µs I0423 20:50:00.567715 3796 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key ] I0423 20:50:00.577444 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29616290-pfkpr, hostIfaceName: 0b2e3d2c24bf4da, network: default, NAD default, SandboxID: "0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24", PCI device ID: , UID: "10335000-5b89-4a66-ac08-75d5cc7504a8", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0423 20:50:00.628352 3796 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:51:34:36:84:a1\",\"name\":\"0b2e3d2c24bf4da\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eac1516e-f499-4db7-8bf0-fec78e055bd6\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0423 20:50:04.408465 3796 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key ] I0423 20:50:04.486720 3796 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr 0b2e3d2c24bf4da66f9773c01e18743c60a8b83b25929d5860558f510866bb24 network default NAD default NAD key default], result "{}", err I0423 20:50:04.531276 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:50:04.531346 3796 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr I0423 20:50:04.532886 3796 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr, ips: 10.128.0.26 I0423 20:50:04.532915 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:50:05.447752 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:50:05.447782 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:50:16.811713 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=67 I0423 20:51:09.937659 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0423 20:51:16.406229 3796 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-l776n for pod on switch ip-10-0-0-172.ec2.internal I0423 20:51:16.406407 3796 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n I0423 20:51:16.424702 3796 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n] pod update took 18.340289ms I0423 20:51:16.424726 3796 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n] addLogicalPort annotation time took 18.369092ms I0423 20:51:16.425901 3796 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n] addLogicalPort took 19.695222ms, libovsdb time 839.872µs I0423 20:51:16.763671 3796 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n a642dd23ad2e763a986b5affcc13eba6619039ec812326fccbc794ddf6ced620 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n a642dd23ad2e763a986b5affcc13eba6619039ec812326fccbc794ddf6ced620 network default NAD default NAD key ] I0423 20:51:16.773487 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-l776n, hostIfaceName: a642dd23ad2e763, network: default, NAD default, SandboxID: "a642dd23ad2e763a986b5affcc13eba6619039ec812326fccbc794ddf6ced620", PCI device ID: , UID: "ed7b061a-08ae-4532-95b4-de0e472172f9", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0423 20:51:16.824140 3796 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n a642dd23ad2e763a986b5affcc13eba6619039ec812326fccbc794ddf6ced620 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-l776n a642dd23ad2e763a986b5affcc13eba6619039ec812326fccbc794ddf6ced620 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:8d:be:d8:64:6f\",\"name\":\"a642dd23ad2e763\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e734af91-d129-4717-a287-418080316137\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0423 20:51:19.695132 3796 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:51:23.820783 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=669 I0423 20:51:24.477623 3796 namespace.go:98] [tsf] adding namespace I0423 20:51:24.478443 3796 namespace.go:102] [tsf] adding namespace took 798.544µs I0423 20:51:24.511818 3796 namespace.go:142] [tsf] updating namespace I0423 20:51:24.527563 3796 namespace.go:142] [tsf] updating namespace W0423 20:51:30.708948 3796 util.go:959] Failed to build node endpoints for node ip-10-0-0-172.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:51:30.744184 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0423 20:51:30.744256 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-v8sd6 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-v8sd6: no pod IPs found I0423 20:51:34.932415 3796 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10