+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-149.ec2.internal ++ [[ -n ip-10-0-1-149.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-149.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0423 19:58:08.304102978 - disable conntrack on geneve port' I0423 19:58:08.304102978 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0423 19:58:08.322487884 - starting ovnkube-node + echo 'I0423 19:58:08.322487884 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-149.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-149.ec2.internal --init-node ip-10-0-1-149.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-149.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0423 19:58:08.368618 3643 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0423 19:58:08.368680 3643 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-d080a8e9ea.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0423 19:58:08.370152 3643 kube.go:431] Waiting for certificate I0423 19:58:08.370181 3643 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0423 19:58:08.370214 3643 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0423 19:58:08.370940 3643 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 19:58:08.400006 3643 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:58:08.400035 3643 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:58:08.402582 3643 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:58:08.402667 3643 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-s6g9n" I0423 19:58:08.412771 3643 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-s6g9n" I0423 19:58:08.412825 3643 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 19:58:09.370970 3643 kube.go:438] Certificate found I0423 19:58:09.371832 3643 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 19:58:09.373196 3643 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0423 19:58:09.420477 3643 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-24 19:53:08 +0000 UTC" deadline="2026-04-24 15:50:26.346402877 +0000 UTC" I0423 19:58:09.420592 3643 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h52m16.925829322s" I0423 19:58:09.444282 3643 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 19:58:09.468366 3643 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0423 19:58:09.468448 3643 server.go:101] MetricServer registers OVS metrics I0423 19:58:09.468759 3643 server.go:105] MetricServer registers OVN DB metrics I0423 19:58:09.470303 3643 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0423 19:58:09.470376 3643 controller_manager.go:370] Starting the ovnkube controller I0423 19:58:09.470393 3643 factory.go:561] Starting watch factory I0423 19:58:09.470390 3643 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-149.ec2.internal I0423 19:58:09.470487 3643 controller_manager.go:395] NBDB zone sync took: 64.928µs I0423 19:58:09.470501 3643 factory.go:561] Starting watch factory I0423 19:58:09.470544 3643 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470571 3643 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470573 3643 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470595 3643 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470761 3643 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470781 3643 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470951 3643 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.470991 3643 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.471260 3643 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.471345 3643 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.471523 3643 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.471612 3643 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.473830 3643 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0423 19:58:09.473873 3643 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0423 19:58:09.488929 3643 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0423 19:58:09.488962 3643 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0423 19:58:09.493061 3643 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0423 19:58:09.493098 3643 server.go:109] MetricServer registers OVN Controller metrics I0423 19:58:09.495014 3643 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.497089 3643 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.497761 3643 server.go:113] MetricServer registers OVN Northd metrics I0423 19:58:09.497851 3643 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.498264 3643 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.498649 3643 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.502722 3643 metrics.go:519] OVN Metrics Server starts to run ... I0423 19:58:09.507818 3643 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 19:58:09.576531 3643 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 19:58:09.576548 3643 factory.go:1977] *v1.Service informer cache synced successfully I0423 19:58:09.576562 3643 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 19:58:09.576566 3643 factory.go:1977] *v1.Pod informer cache synced successfully I0423 19:58:09.576570 3643 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 19:58:09.576556 3643 factory.go:1977] *v1.Node informer cache synced successfully I0423 19:58:09.576599 3643 factory.go:1977] *v1.Service informer cache synced successfully I0423 19:58:09.576610 3643 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 19:58:09.576615 3643 factory.go:1977] *v1.Pod informer cache synced successfully I0423 19:58:09.576620 3643 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 19:58:09.576573 3643 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 19:58:09.576638 3643 factory.go:1977] *v1.Node informer cache synced successfully I0423 19:58:09.576735 3643 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:58:09.576759 3643 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:58:09.576760 3643 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:58:09.576772 3643 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:58:09.580357 3643 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:58:09.580368 3643 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 19:58:09.582232 3643 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 19:58:09.582246 3643 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 19:58:09.582254 3643 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 19:58:09.582247 3643 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 19:58:09.582335 3643 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.582349 3643 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.585427 3643 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.587431 3643 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 19:58:09.587517 3643 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.587537 3643 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.587583 3643 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 19:58:09.590630 3643 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.593316 3643 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 19:58:09.593335 3643 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 19:58:09.593387 3643 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.593399 3643 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.596432 3643 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.598366 3643 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 19:58:09.598380 3643 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 19:58:09.598435 3643 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.598447 3643 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.601358 3643 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.603968 3643 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 19:58:09.604002 3643 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 19:58:09.604021 3643 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.604031 3643 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.606854 3643 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.609548 3643 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 19:58:09.609559 3643 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 19:58:09.609736 3643 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:58:09.609749 3643 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:58:09.612662 3643 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 19:58:09.615339 3643 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 19:58:09.615356 3643 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 19:58:09.615429 3643 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.615436 3643 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.615449 3643 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.615450 3643 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.618648 3643 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.618682 3643 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 19:58:09.620676 3643 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 19:58:09.620687 3643 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 19:58:09.620694 3643 factory.go:680] Watch Factory start up complete, took: 150.306467ms I0423 19:58:09.620694 3643 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 19:58:09.620702 3643 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 19:58:09.620708 3643 factory.go:680] Watch Factory start up complete, took: 150.205957ms I0423 19:58:09.620715 3643 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-149.ec2.internal" zone I0423 19:58:09.620758 3643 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0423 19:58:09.661890 3643 default_node_network_controller.go:710] Initializing the default node network controller I0423 19:58:09.759290 3643 udn_isolation.go:101] Starting UDN host isolation manager I0423 19:58:09.765187 3643 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0423 19:58:09.846502 3643 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0423 19:58:09.846608 3643 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0423 19:58:09.846628 3643 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0423 19:58:09.880437 3643 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0423 19:58:09.880518 3643 default_node_network_controller.go:809] Node ip-10-0-1-149.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0423 19:58:09.886731 3643 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.149"] k8s.ovn.org/zone-name:ip-10-0-1-149.ec2.internal] on node ip-10-0-1-149.ec2.internal I0423 19:58:09.910292 3643 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0423 19:58:09.917286 3643 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0423 19:58:09.917451 3643 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0423 19:58:09.917582 3643 gateway_init.go:263] Preparing Gateway I0423 19:58:09.917596 3643 gateway_shared_intf.go:1687] Creating new gateway I0423 19:58:10.031512 3643 iptables.go:108] Creating table: filter chain: FORWARD I0423 19:58:10.038730 3643 iptables.go:108] Creating table: filter chain: OUTPUT I0423 19:58:10.045789 3643 gateway_shared_intf.go:1816] Gateway Creation Complete I0423 19:58:10.045809 3643 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-149.ec2.internal","mac-address":"0a:ff:ea:25:56:99","ip-addresses":["10.0.1.149/24"],"ip-address":"10.0.1.149/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:8a0f73b2-62dc-44ec-8807-2f51c6f0bfb5 k8s.ovn.org/node-encap-ips:["10.0.1.149"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.149/24"} k8s.ovn.org/zone-name:ip-10-0-1-149.ec2.internal] on node ip-10-0-1-149.ec2.internal I0423 19:58:10.067716 3643 default_node_network_controller.go:519] Node connection status = connected I0423 19:58:10.121541 3643 controller_manager.go:429] Waiting for node in zone sync took: 500.811731ms I0423 19:58:10.127090 3643 controller_manager.go:611] Switching to transit router for layer2 networks I0423 19:58:10.127106 3643 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-149.ec2.internal I0423 19:58:10.145348 3643 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 19:58:10.153112 3643 controller_manager.go:290] SCTP support detected in OVN I0423 19:58:10.153730 3643 services_controller.go:69] Creating services controller for network=default I0423 19:58:10.153834 3643 udn_enabled_svc.go:80] Waiting for service informer to sync I0423 19:58:10.153881 3643 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0423 19:58:10.153917 3643 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0423 19:58:10.153929 3643 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0423 19:58:10.153938 3643 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 19:58:10.154136 3643 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0423 19:58:10.154182 3643 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0423 19:58:10.154248 3643 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 73.069µs I0423 19:58:10.154287 3643 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 97.229µs I0423 19:58:10.154295 3643 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 112.807µs I0423 19:58:10.154300 3643 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0423 19:58:10.154306 3643 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0423 19:58:10.154316 3643 default_network_controller.go:325] Starting the default network controller I0423 19:58:10.154350 3643 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.905µs I0423 19:58:10.154377 3643 udn_enabled_svc.go:95] Performing full resync I0423 19:58:10.154725 3643 udn_enabled_svc.go:99] Waiting for handler to sync I0423 19:58:10.154737 3643 udn_enabled_svc.go:104] Starting worker I0423 19:58:10.164906 3643 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0423 19:58:10.164986 3643 repair.go:33] Syncing exgw routes took 64.596µs I0423 19:58:10.165019 3643 default_network_controller.go:414] Starting all the Watchers... I0423 19:58:10.165190 3643 namespace.go:98] [openshift-apiserver] adding namespace I0423 19:58:10.165192 3643 namespace.go:98] [openshift-kube-scheduler] adding namespace I0423 19:58:10.165199 3643 namespace.go:98] [openshift-config-managed] adding namespace I0423 19:58:10.165215 3643 namespace.go:98] [openshift-infra] adding namespace I0423 19:58:10.165222 3643 namespace.go:98] [openshift-host-network] adding namespace I0423 19:58:10.165231 3643 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0423 19:58:10.165232 3643 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0423 19:58:10.165247 3643 namespace.go:98] [openshift-ingress-canary] adding namespace I0423 19:58:10.165253 3643 namespace.go:98] [openshift-etcd] adding namespace I0423 19:58:10.165259 3643 namespace.go:98] [openshift-machine-api] adding namespace I0423 19:58:10.165304 3643 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0423 19:58:10.165313 3643 namespace.go:98] [openshift-cluster-version] adding namespace I0423 19:58:10.165686 3643 namespace.go:98] [openshift-route-controller-manager] adding namespace I0423 19:58:10.165695 3643 namespace.go:98] [openshift-service-ca-operator] adding namespace I0423 19:58:10.165725 3643 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0423 19:58:10.165946 3643 namespace.go:102] [openshift-apiserver] adding namespace took 742.975µs I0423 19:58:10.165962 3643 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0423 19:58:10.166545 3643 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 576.217µs I0423 19:58:10.166563 3643 namespace.go:98] [openshift-backplane] adding namespace I0423 19:58:10.167134 3643 namespace.go:102] [openshift-backplane] adding namespace took 561.95µs I0423 19:58:10.167156 3643 namespace.go:98] [openshift-network-operator] adding namespace I0423 19:58:10.167711 3643 namespace.go:102] [openshift-kube-scheduler] adding namespace took 2.503589ms I0423 19:58:10.167727 3643 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0423 19:58:10.168170 3643 namespace.go:102] [openshift-config-managed] adding namespace took 2.959104ms I0423 19:58:10.168186 3643 namespace.go:98] [kube-public] adding namespace I0423 19:58:10.168772 3643 namespace.go:102] [openshift-infra] adding namespace took 3.546989ms I0423 19:58:10.168787 3643 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0423 19:58:10.169406 3643 namespace.go:102] [openshift-host-network] adding namespace took 4.176784ms I0423 19:58:10.169418 3643 namespace.go:98] [openshift-network-node-identity] adding namespace I0423 19:58:10.170004 3643 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 4.765278ms I0423 19:58:10.170023 3643 namespace.go:98] [openshift-operators-redhat] adding namespace I0423 19:58:10.170517 3643 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 5.277512ms I0423 19:58:10.170533 3643 namespace.go:98] [openshift-backplane-cse] adding namespace I0423 19:58:10.171046 3643 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.792077ms I0423 19:58:10.171061 3643 namespace.go:98] [openshift-insights] adding namespace I0423 19:58:10.171612 3643 namespace.go:102] [openshift-etcd] adding namespace took 6.353927ms I0423 19:58:10.171626 3643 namespace.go:98] [openshift-backplane-srep] adding namespace I0423 19:58:10.172129 3643 namespace.go:102] [openshift-machine-api] adding namespace took 6.858888ms I0423 19:58:10.172144 3643 namespace.go:98] [openshift-apiserver-operator] adding namespace I0423 19:58:10.172670 3643 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.343682ms I0423 19:58:10.172686 3643 namespace.go:98] [openshift-backplane-mobb] adding namespace I0423 19:58:10.173227 3643 namespace.go:102] [openshift-cluster-version] adding namespace took 7.898466ms I0423 19:58:10.173242 3643 namespace.go:98] [openshift-dns-operator] adding namespace I0423 19:58:10.173825 3643 namespace.go:102] [openshift-route-controller-manager] adding namespace took 8.125996ms I0423 19:58:10.173848 3643 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0423 19:58:10.174475 3643 namespace.go:102] [openshift-service-ca-operator] adding namespace took 8.774373ms I0423 19:58:10.174486 3643 namespace.go:98] [openshift-ingress-operator] adding namespace I0423 19:58:10.175002 3643 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 9.26244ms I0423 19:58:10.175018 3643 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0423 19:58:10.175583 3643 namespace.go:102] [openshift-network-operator] adding namespace took 8.419874ms I0423 19:58:10.175598 3643 namespace.go:98] [openshift-must-gather-operator] adding namespace I0423 19:58:10.176065 3643 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 8.331974ms I0423 19:58:10.176080 3643 namespace.go:98] [default] adding namespace I0423 19:58:10.176595 3643 namespace.go:102] [kube-public] adding namespace took 8.400817ms I0423 19:58:10.176610 3643 namespace.go:98] [openshift-marketplace] adding namespace I0423 19:58:10.177130 3643 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.337738ms I0423 19:58:10.177142 3643 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0423 19:58:10.177667 3643 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.243072ms I0423 19:58:10.177685 3643 namespace.go:98] [openshift-machine-config-operator] adding namespace I0423 19:58:10.178208 3643 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.179646ms I0423 19:58:10.178222 3643 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0423 19:58:10.178716 3643 namespace.go:102] [openshift-backplane-cse] adding namespace took 8.17917ms I0423 19:58:10.178727 3643 namespace.go:98] [openshift-customer-monitoring] adding namespace I0423 19:58:10.179233 3643 namespace.go:102] [openshift-insights] adding namespace took 8.167437ms I0423 19:58:10.179245 3643 namespace.go:98] [openshift-console-user-settings] adding namespace I0423 19:58:10.179725 3643 namespace.go:102] [openshift-backplane-srep] adding namespace took 8.093099ms I0423 19:58:10.179740 3643 namespace.go:98] [dedicated-admin] adding namespace I0423 19:58:10.180223 3643 namespace.go:102] [openshift-apiserver-operator] adding namespace took 8.073456ms I0423 19:58:10.180238 3643 namespace.go:98] [openshift-config-operator] adding namespace I0423 19:58:10.180746 3643 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.048261ms I0423 19:58:10.180760 3643 namespace.go:98] [openshift-node] adding namespace I0423 19:58:10.181238 3643 namespace.go:102] [openshift-dns-operator] adding namespace took 7.990837ms I0423 19:58:10.181247 3643 namespace.go:98] [openshift-authentication] adding namespace I0423 19:58:10.181813 3643 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.959925ms I0423 19:58:10.181824 3643 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0423 19:58:10.182295 3643 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.803967ms I0423 19:58:10.182309 3643 namespace.go:98] [openshift] adding namespace I0423 19:58:10.182864 3643 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.839944ms I0423 19:58:10.182876 3643 namespace.go:98] [openshift-backplane-csm] adding namespace I0423 19:58:10.183369 3643 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.766556ms I0423 19:58:10.183380 3643 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0423 19:58:10.183876 3643 namespace.go:102] [default] adding namespace took 7.792165ms I0423 19:58:10.183885 3643 namespace.go:98] [openshift-network-console] adding namespace I0423 19:58:10.184397 3643 namespace.go:102] [openshift-marketplace] adding namespace took 7.782165ms I0423 19:58:10.184407 3643 namespace.go:98] [openshift-console-operator] adding namespace I0423 19:58:10.184945 3643 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.799368ms I0423 19:58:10.184954 3643 namespace.go:98] [openshift-multus] adding namespace I0423 19:58:10.185474 3643 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.782906ms I0423 19:58:10.185488 3643 namespace.go:98] [openshift-backplane-tam] adding namespace I0423 19:58:10.186009 3643 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.781734ms I0423 19:58:10.186022 3643 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0423 19:58:10.186551 3643 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.819123ms I0423 19:58:10.186567 3643 namespace.go:98] [openshift-authentication-operator] adding namespace I0423 19:58:10.187031 3643 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.782018ms I0423 19:58:10.187042 3643 namespace.go:98] [openshift-ingress] adding namespace I0423 19:58:10.187550 3643 namespace.go:102] [dedicated-admin] adding namespace took 7.805865ms I0423 19:58:10.187562 3643 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0423 19:58:10.188076 3643 namespace.go:102] [openshift-config-operator] adding namespace took 7.833262ms I0423 19:58:10.188087 3643 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0423 19:58:10.188633 3643 namespace.go:102] [openshift-node] adding namespace took 7.868397ms I0423 19:58:10.188645 3643 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0423 19:58:10.189179 3643 namespace.go:102] [openshift-authentication] adding namespace took 7.927801ms I0423 19:58:10.189190 3643 namespace.go:98] [openshift-kube-apiserver] adding namespace I0423 19:58:10.189747 3643 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.918206ms I0423 19:58:10.189763 3643 namespace.go:98] [openshift-operators] adding namespace I0423 19:58:10.190256 3643 namespace.go:102] [openshift] adding namespace took 7.942289ms I0423 19:58:10.190281 3643 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0423 19:58:10.190885 3643 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.003389ms I0423 19:58:10.190900 3643 namespace.go:98] [openshift-dns] adding namespace I0423 19:58:10.191447 3643 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.055948ms I0423 19:58:10.192093 3643 namespace.go:102] [openshift-network-console] adding namespace took 8.201087ms I0423 19:58:10.192108 3643 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0423 19:58:10.192706 3643 namespace.go:102] [openshift-console-operator] adding namespace took 8.291971ms W0423 19:58:10.192716 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-5z6fd: pod openshift-multus/network-metrics-daemon-5z6fd: no pod IPs found I0423 19:58:10.192723 3643 namespace.go:98] [open-cluster-management-2ps2fssbndokb6i5u6qmofj8dp8f08d0] adding namespace W0423 19:58:10.192731 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-cgz9p: pod openshift-multus/network-metrics-daemon-cgz9p: no pod IPs found I0423 19:58:10.193376 3643 namespace.go:102] [openshift-multus] adding namespace took 8.417721ms I0423 19:58:10.193393 3643 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0423 19:58:10.193891 3643 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.39734ms I0423 19:58:10.193908 3643 namespace.go:98] [openshift-backplane-cee] adding namespace I0423 19:58:10.194412 3643 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 8.385277ms I0423 19:58:10.194423 3643 namespace.go:98] [kube-node-lease] adding namespace I0423 19:58:10.194928 3643 namespace.go:102] [openshift-authentication-operator] adding namespace took 8.352536ms I0423 19:58:10.194944 3643 namespace.go:98] [kube-system] adding namespace I0423 19:58:10.195541 3643 namespace.go:102] [openshift-ingress] adding namespace took 8.494495ms I0423 19:58:10.195551 3643 namespace.go:98] [openshift-controller-manager] adding namespace I0423 19:58:10.196456 3643 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.887939ms I0423 19:58:10.196473 3643 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0423 19:58:10.196868 3643 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.772471ms I0423 19:58:10.196891 3643 namespace.go:98] [openshift-image-registry] adding namespace I0423 19:58:10.197462 3643 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.811812ms I0423 19:58:10.198012 3643 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.816527ms I0423 19:58:10.198026 3643 namespace.go:98] [openshift-network-diagnostics] adding namespace I0423 19:58:10.198537 3643 namespace.go:102] [openshift-operators] adding namespace took 8.768595ms I0423 19:58:10.198554 3643 namespace.go:98] [openshift-logging] adding namespace I0423 19:58:10.199040 3643 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.750766ms I0423 19:58:10.199060 3643 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0423 19:58:10.199597 3643 namespace.go:102] [openshift-dns] adding namespace took 8.691705ms I0423 19:58:10.199613 3643 namespace.go:98] [openshift-package-operator] adding namespace I0423 19:58:10.200226 3643 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 8.111883ms I0423 19:58:10.200242 3643 namespace.go:98] [openshift-config] adding namespace I0423 19:58:10.200854 3643 namespace.go:102] [open-cluster-management-2ps2fssbndokb6i5u6qmofj8dp8f08d0] adding namespace took 8.124867ms I0423 19:58:10.201382 3643 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.980594ms I0423 19:58:10.202043 3643 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.127594ms I0423 19:58:10.202725 3643 namespace.go:102] [kube-node-lease] adding namespace took 8.295508ms I0423 19:58:10.203203 3643 namespace.go:102] [kube-system] adding namespace took 8.253213ms I0423 19:58:10.203678 3643 namespace.go:102] [openshift-controller-manager] adding namespace took 8.120516ms I0423 19:58:10.204181 3643 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.701161ms I0423 19:58:10.204197 3643 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0423 19:58:10.204662 3643 namespace.go:102] [openshift-image-registry] adding namespace took 7.76344ms I0423 19:58:10.204678 3643 namespace.go:98] [openshift-console] adding namespace W0423 19:58:10.204687 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-xswps: pod openshift-network-diagnostics/network-check-target-xswps: no pod IPs found W0423 19:58:10.204711 3643 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-5ph4b: pod openshift-network-diagnostics/network-check-target-5ph4b: no pod IPs found I0423 19:58:10.205308 3643 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.276643ms I0423 19:58:10.205322 3643 namespace.go:98] [openshift-monitoring] adding namespace I0423 19:58:10.205759 3643 namespace.go:102] [openshift-logging] adding namespace took 7.197818ms I0423 19:58:10.206264 3643 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.198639ms I0423 19:58:10.206789 3643 namespace.go:102] [openshift-package-operator] adding namespace took 7.169918ms I0423 19:58:10.207372 3643 namespace.go:102] [openshift-config] adding namespace took 7.123653ms I0423 19:58:10.208009 3643 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 3.808143ms I0423 19:58:10.208492 3643 namespace.go:102] [openshift-console] adding namespace took 3.810293ms I0423 19:58:10.209050 3643 namespace.go:102] [openshift-monitoring] adding namespace took 3.722166ms W0423 19:58:10.209134 3643 master.go:471] Unable to parse remote node ip-10-0-0-111.ec2.internal chassis-id annotation. Chassis may be removed during sync I0423 19:58:10.209250 3643 master.go:548] Adding or Updating local node "ip-10-0-1-149.ec2.internal" for network "default" I0423 19:58:10.209324 3643 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-111.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-111.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-111.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-111.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-111.ec2.internal I0423 19:58:10.210245 3643 zone_ic_handler.go:195] Time taken to create transit switch: 321.476µs I0423 19:58:10.210257 3643 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-102.ec2.internal for the network default I0423 19:58:10.211399 3643 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-102.ec2.internal" on network "default" took: 1.485153ms W0423 19:58:10.217987 3643 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-xswps: pod openshift-network-diagnostics/network-check-target-xswps: no pod IPs found W0423 19:58:10.218006 3643 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-cgz9p: pod openshift-multus/network-metrics-daemon-cgz9p: no pod IPs found I0423 19:58:10.219128 3643 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-149.ec2.internal for the network default I0423 19:58:10.219364 3643 zone_ic_handler.go:195] Time taken to create transit switch: 220.013µs I0423 19:58:10.220263 3643 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0423 19:58:10.220314 3643 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0423 19:58:10.220326 3643 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0423 19:58:10.220355 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-111.ec2.internal in network "default" I0423 19:58:10.220395 3643 node_tracker.go:235] Node ip-10-0-0-111.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-111.ec2.internal" I0423 19:58:10.220413 3643 node_tracker.go:169] Node ip-10-0-0-111.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:10.220431 3643 services_controller.go:574] Full service sync requested for network=default I0423 19:58:10.220446 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-149.ec2.internal in network "default" I0423 19:58:10.220481 3643 node_tracker.go:169] Node ip-10-0-1-149.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:10.220490 3643 services_controller.go:574] Full service sync requested for network=default I0423 19:58:10.220499 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-102.ec2.internal in network "default" I0423 19:58:10.220531 3643 node_tracker.go:169] Node ip-10-0-2-102.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:10.220540 3643 services_controller.go:574] Full service sync requested for network=default I0423 19:58:10.320865 3643 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0423 19:58:10.320899 3643 services_controller.go:208] Setting up event handlers for services for network=default I0423 19:58:10.320974 3643 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0423 19:58:10.321035 3643 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0423 19:58:10.321059 3643 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0423 19:58:10.421242 3643 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0423 19:58:10.421265 3643 repair.go:60] Starting repairing loop for services I0423 19:58:10.421364 3643 repair.go:131] Deleted 0 stale service LBs I0423 19:58:10.421382 3643 repair.go:137] Deleted 0 stale Chassis Template Vars I0423 19:58:10.421401 3643 repair.go:62] Finished repairing loop for services: 136.021µs I0423 19:58:10.421424 3643 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0423 19:58:10.421433 3643 services_controller.go:254] Starting workers for network=default I0423 19:58:10.421817 3643 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-5ph4b, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5ph4b: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5ph4b: no pod IPs found I0423 19:58:10.421833 3643 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xswps] creating logical port openshift-network-diagnostics_network-check-target-xswps for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:10.421833 3643 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-5z6fd, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5z6fd: suppressed error logged: pod openshift-multus/network-metrics-daemon-5z6fd: no pod IPs found I0423 19:58:10.421943 3643 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-cgz9p] creating logical port openshift-multus_network-metrics-daemon-cgz9p for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:10.421968 3643 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-xswps I0423 19:58:10.422081 3643 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-cgz9p I0423 19:58:10.442596 3643 pod.go:62] [openshift-multus/network-metrics-daemon-cgz9p] pod update took 20.545946ms I0423 19:58:10.442617 3643 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-cgz9p] addLogicalPort annotation time took 20.56813ms I0423 19:58:10.443631 3643 pods.go:271] [openshift-multus/network-metrics-daemon-cgz9p] addLogicalPort took 21.694565ms, libovsdb time 827.6µs I0423 19:58:10.446145 3643 pod.go:62] [openshift-network-diagnostics/network-check-target-xswps] pod update took 24.23475ms I0423 19:58:10.446165 3643 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-xswps] addLogicalPort annotation time took 24.257806ms I0423 19:58:10.446969 3643 pods.go:271] [openshift-network-diagnostics/network-check-target-xswps] addLogicalPort took 25.14872ms, libovsdb time 563.41µs I0423 19:58:10.447332 3643 admin_network_policy_controller.go:227] Starting controller default-network-controller I0423 19:58:10.447357 3643 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0423 19:58:10.447379 3643 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0423 19:58:10.447387 3643 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0423 19:58:10.447580 3643 repair.go:29] Repairing admin network policies took 187.605µs I0423 19:58:10.447693 3643 repair.go:92] Repairing baseline admin network policies took 103.442µs I0423 19:58:10.448073 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0423 19:58:10.448090 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0423 19:58:10.449665 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0423 19:58:10.449686 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0423 19:58:10.449696 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0423 19:58:10.451574 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0423 19:58:10.451594 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0423 19:58:10.451606 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0423 19:58:10.453647 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0423 19:58:10.453665 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0423 19:58:10.453671 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0423 19:58:10.453945 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0423 19:58:10.453961 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0423 19:58:10.453968 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0423 19:58:10.455331 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0423 19:58:10.455351 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0423 19:58:10.455359 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0423 19:58:10.456888 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0423 19:58:10.456906 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0423 19:58:10.456915 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0423 19:58:10.457423 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0423 19:58:10.457443 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0423 19:58:10.457452 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0423 19:58:10.459406 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0423 19:58:10.459428 3643 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0423 19:58:10.459441 3643 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0423 19:58:10.461545 3643 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0423 19:58:10.461564 3643 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0423 19:58:10.461572 3643 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0423 19:58:10.462545 3643 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0423 19:58:10.462568 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0423 19:58:10.462580 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0423 19:58:10.463078 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0423 19:58:10.463095 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0423 19:58:10.463103 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0423 19:58:10.463651 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0423 19:58:10.463665 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0423 19:58:10.464449 3643 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0423 19:58:10.464463 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 19:58:10.465150 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0423 19:58:10.465171 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0423 19:58:10.465184 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0423 19:58:10.466657 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0423 19:58:10.466681 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0423 19:58:10.466693 3643 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0423 19:58:10.468476 3643 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0423 19:58:10.468495 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0423 19:58:10.468503 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0423 19:58:10.469236 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0423 19:58:10.469256 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0423 19:58:10.469280 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0423 19:58:10.469788 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0423 19:58:10.469804 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0423 19:58:10.469814 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0423 19:58:10.470997 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0423 19:58:10.471018 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0423 19:58:10.471027 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0423 19:58:10.471377 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0423 19:58:10.471396 3643 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0423 19:58:10.471406 3643 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0423 19:58:10.472533 3643 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0423 19:58:10.472553 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0423 19:58:10.472561 3643 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0423 19:58:10.473013 3643 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0423 19:58:10.473031 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0423 19:58:10.473038 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0423 19:58:10.473558 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0423 19:58:10.473572 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0423 19:58:10.473579 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0423 19:58:10.474113 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0423 19:58:10.474131 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0423 19:58:10.474153 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 19:58:10.474884 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0423 19:58:10.474902 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0423 19:58:10.474912 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0423 19:58:10.476003 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0423 19:58:10.476020 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0423 19:58:10.476028 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0423 19:58:10.476624 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0423 19:58:10.476641 3643 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0423 19:58:10.476648 3643 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0423 19:58:10.477188 3643 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0423 19:58:10.477204 3643 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0423 19:58:10.477211 3643 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0423 19:58:10.477537 3643 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0423 19:58:10.477554 3643 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0423 19:58:10.477564 3643 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0423 19:58:10.477848 3643 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0423 19:58:10.483677 3643 egressfirewall.go:374] Starting EgressFirewall controller I0423 19:58:10.483707 3643 controller.go:133] Adding controller egress-firewall-controller event handlers I0423 19:58:10.483712 3643 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0423 19:58:10.483733 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0423 19:58:10.483736 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0423 19:58:10.483744 3643 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0423 19:58:10.483746 3643 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0423 19:58:10.483992 3643 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0423 19:58:10.484007 3643 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0423 19:58:10.484012 3643 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0423 19:58:10.484021 3643 egressqos.go:193] Setting up event handlers for EgressQoS I0423 19:58:10.484032 3643 egressfirewall.go:723] Syncing node "ip-10-0-1-149.ec2.internal" for egress firewall I0423 19:58:10.484047 3643 egressfirewall.go:723] Syncing node "ip-10-0-2-102.ec2.internal" for egress firewall I0423 19:58:10.484053 3643 egressfirewall.go:723] Syncing node "ip-10-0-0-111.ec2.internal" for egress firewall I0423 19:58:10.484139 3643 egressqos.go:245] Starting EgressQoS Controller I0423 19:58:10.484152 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0423 19:58:10.484160 3643 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0423 19:58:10.484165 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0423 19:58:10.484170 3643 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0423 19:58:10.484174 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0423 19:58:10.484179 3643 shared_informer.go:356] "Caches are synced" controller="egressqos" I0423 19:58:10.484182 3643 egressqos.go:259] Repairing EgressQoSes I0423 19:58:10.484186 3643 egressqos.go:399] Starting repairing loop for egressqos I0423 19:58:10.484257 3643 egressqos.go:401] Finished repairing loop for egressqos: 70.8µs I0423 19:58:10.484285 3643 egressservice_zone.go:132] Setting up event handlers for Egress Services I0423 19:58:10.484305 3643 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-149.ec2.internal I0423 19:58:10.484353 3643 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-149.ec2.internal : 49.558µs I0423 19:58:10.484407 3643 egressservice_zone.go:210] Starting Egress Services Controller I0423 19:58:10.484417 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 19:58:10.484423 3643 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 19:58:10.484429 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 19:58:10.484434 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 19:58:10.484439 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 19:58:10.484443 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 19:58:10.484448 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0423 19:58:10.484452 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0423 19:58:10.484456 3643 egressservice_zone.go:228] Repairing Egress Services I0423 19:58:10.484730 3643 master_controller.go:88] Starting Admin Policy Based Route Controller I0423 19:58:10.484741 3643 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 19:58:10.484800 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-149.ec2.internal I0423 19:58:10.484811 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-149.ec2.internal: 12.891µs I0423 19:58:10.484818 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-102.ec2.internal I0423 19:58:10.484827 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-102.ec2.internal: 9.469µs I0423 19:58:10.484831 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-111.ec2.internal I0423 19:58:10.484835 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-111.ec2.internal: 3.797µs I0423 19:58:10.484925 3643 default_network_controller.go:585] Completing all the Watchers took 319.895117ms I0423 19:58:10.484938 3643 default_network_controller.go:589] Starting unidling controllers I0423 19:58:10.484949 3643 unidle.go:46] Registering OVN SB ControllerEvent handler I0423 19:58:10.484958 3643 unidle.go:63] Populating Initial ContollerEvent events I0423 19:58:10.484969 3643 unidle.go:79] Setting up event handlers for services I0423 19:58:10.485014 3643 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0423 19:58:10.567842 3643 default_node_network_controller.go:519] Node connection status = connected I0423 19:58:11.068357 3643 default_node_network_controller.go:519] Node connection status = connected I0423 19:58:11.567761 3643 default_node_network_controller.go:519] Node connection status = connected I0423 19:58:12.068165 3643 default_node_network_controller.go:519] Node connection status = connected I0423 19:58:12.311092 3643 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0423 19:58:12.568006 3643 default_node_network_controller.go:519] Node connection status = connected I0423 19:58:12.586170 3643 bridgeconfig.go:537] Gateway is ready I0423 19:58:12.586189 3643 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0423 19:58:12.597947 3643 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.149. Updating OVN encap IP. I0423 19:58:12.603766 3643 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.149 - it is already configured I0423 19:58:12.603784 3643 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.149/24:{} 172.20.0.1/32:{}]. Updating annotations. I0423 19:58:12.604018 3643 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.149/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-149.ec2.internal","mac-address":"0a:ff:ea:25:56:99","ip-addresses":["10.0.1.149/24"],"ip-address":"10.0.1.149/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:8a0f73b2-62dc-44ec-8807-2f51c6f0bfb5 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.149/24"}] on node ip-10-0-1-149.ec2.internal I0423 19:58:12.620471 3643 egressfirewall.go:723] Syncing node "ip-10-0-1-149.ec2.internal" for egress firewall I0423 19:58:12.620502 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-149.ec2.internal I0423 19:58:12.620504 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-149.ec2.internal in network "default" I0423 19:58:12.620515 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-149.ec2.internal: 17.438µs I0423 19:58:12.620576 3643 master.go:548] Adding or Updating local node "ip-10-0-1-149.ec2.internal" for network "default" I0423 19:58:12.620585 3643 node_tracker.go:169] Node ip-10-0-1-149.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:12.620639 3643 services_controller.go:574] Full service sync requested for network=default I0423 19:58:12.621096 3643 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.149 I0423 19:58:12.621176 3643 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-149.ec2.internal I0423 19:58:12.625621 3643 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-149.ec2.internal. Updating no re-route policies I0423 19:58:12.625692 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-9gd6h I0423 19:58:12.625706 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-n9rmf I0423 19:58:12.625711 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-9gd6h I0423 19:58:12.625726 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-n9rmf I0423 19:58:12.625729 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-kzptb I0423 19:58:12.625740 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-9gd6h after 0 failed attempt(s) I0423 19:58:12.625738 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-t7xcd I0423 19:58:12.625746 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-n9rmf after 0 failed attempt(s) I0423 19:58:12.625751 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-kzptb I0423 19:58:12.625754 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-5ph4b I0423 19:58:12.625753 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-t7xcd I0423 19:58:12.625761 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-5ph4b I0423 19:58:12.625760 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xswps I0423 19:58:12.625773 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-149.ec2.internal I0423 19:58:12.625779 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-kzptb after 0 failed attempt(s) I0423 19:58:12.625776 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-t7xcd after 0 failed attempt(s) I0423 19:58:12.625781 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xswps I0423 19:58:12.625788 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-149.ec2.internal I0423 19:58:12.625793 3643 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-5ph4b, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5ph4b: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5ph4b: no pod IPs found I0423 19:58:12.625796 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-j9v4c I0423 19:58:12.625798 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-5z6fd I0423 19:58:12.625802 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-j9v4c I0423 19:58:12.625806 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-149.ec2.internal after 0 failed attempt(s) I0423 19:58:12.625803 3643 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-149.ec2.internal I0423 19:58:12.625809 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-5z6fd I0423 19:58:12.625814 3643 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-149.ec2.internal I0423 19:58:12.625816 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-j9v4c after 0 failed attempt(s) I0423 19:58:12.625819 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-s7vv8 I0423 19:58:12.625823 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7kc24 I0423 19:58:12.625824 3643 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-dkngw I0423 19:58:12.625824 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-q9qbw I0423 19:58:12.625826 3643 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-149.ec2.internal after 0 failed attempt(s) I0423 19:58:12.625830 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7kc24 I0423 19:58:12.625833 3643 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-xswps] creating logical port openshift-network-diagnostics_network-check-target-xswps for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:12.625834 3643 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-dkngw I0423 19:58:12.625834 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-q9qbw I0423 19:58:12.625844 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7kc24 after 0 failed attempt(s) I0423 19:58:12.625843 3643 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-5z6fd, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5z6fd: suppressed error logged: pod openshift-multus/network-metrics-daemon-5z6fd: no pod IPs found I0423 19:58:12.625849 3643 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-cgz9p I0423 19:58:12.625855 3643 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-dkngw after 0 failed attempt(s) I0423 19:58:12.625857 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-q9qbw after 0 failed attempt(s) I0423 19:58:12.625862 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-cgz9p I0423 19:58:12.625827 3643 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-s7vv8 I0423 19:58:12.625876 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-s7vv8 after 0 failed attempt(s) I0423 19:58:12.625892 3643 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-cgz9p] creating logical port openshift-multus_network-metrics-daemon-cgz9p for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:12.626666 3643 pods.go:271] [openshift-network-diagnostics/network-check-target-xswps] addLogicalPort took 846.597µs, libovsdb time 279.518µs I0423 19:58:12.626682 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-xswps after 0 failed attempt(s) I0423 19:58:12.626732 3643 pods.go:271] [openshift-multus/network-metrics-daemon-cgz9p] addLogicalPort took 851.861µs, libovsdb time 338.781µs I0423 19:58:12.626741 3643 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-cgz9p after 0 failed attempt(s) I0423 19:58:12.644453 3643 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0423 19:58:12.657458 3643 iptables.go:108] Creating table: mangle chain: OUTPUT I0423 19:58:12.661139 3643 iptables.go:108] Creating table: nat chain: OUTPUT I0423 19:58:12.693723 3643 iptables.go:108] Creating table: nat chain: PREROUTING I0423 19:58:12.742134 3643 iptables.go:108] Creating table: filter chain: FORWARD I0423 19:58:12.760154 3643 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0423 19:58:12.760260 3643 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0423 19:58:12.760330 3643 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0423 19:58:12.760443 3643 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0423 19:58:12.760506 3643 gateway_init.go:325] Gateway and management port readiness took 2.69731797s I0423 19:58:12.760540 3643 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0423 19:58:12.760581 3643 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0423 19:58:12.760599 3643 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0423 19:58:12.760612 3643 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 19:58:12.760682 3643 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0423 19:58:12.760765 3643 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 48.697µs I0423 19:58:12.761030 3643 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0423 19:58:12.761058 3643 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 19.207µs I0423 19:58:12.761068 3643 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 38.159µs I0423 19:58:12.761073 3643 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0423 19:58:12.761085 3643 nad_controller.go:379] [node-nad-controller NAD controller]: started I0423 19:58:12.761094 3643 default_node_network_controller.go:899] Starting the default node network controller I0423 19:58:12.761101 3643 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0423 19:58:12.761197 3643 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 18.937µs I0423 19:58:12.761379 3643 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0423 19:58:12.761388 3643 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0423 19:58:12.761419 3643 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc2490 0xc0002ee380 0xc0002ee3f0 0xc0002ee460 0xc0002ee4d0 0xc0002ee5b0 0xc0002ee620 0xc000be2a00 0xc0002ee690 0xc0002ee700 0xc000aa0370 0xc0065b9570 0xc0002ee770 map[0x2ff9080:0xc002a93da0 0x2ff95c0:0xc002a93f20 0x2ffa2e0:0xc002616e40 0x2ffa580:0xc002a93d40 0x2ffad60:0xc002a93e00 0x2ffb000:0xc002a93e60 0x2ffb2a0:0xc002a93ec0 0x2ffba80:0xc0066ca000 0x2ffbd20:0xc0066ca060 0x3026980:0xc001707860 0x3026d20:0xc001707a40 0x3027800:0xc000bbde60 0x3029160:0xc001c0e000 0x30298a0:0xc001707aa0 0x302a720:0xc0010163c0] 0xc0006beee0 0} I0423 19:58:12.761482 3643 gateway.go:151] Starting gateway service sync I0423 19:58:12.761918 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0423 19:58:12.764403 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0423 19:58:12.780877 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0423 19:58:12.782186 3643 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0423 19:58:12.783450 3643 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0423 19:58:12.817279 3643 gateway.go:168] Gateway service sync done. Time taken: 55.77144ms I0423 19:58:12.817923 3643 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31856 I0423 19:58:12.818110 3643 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc2490 0xc0002ee380 0xc0002ee3f0 0xc0002ee460 0xc0002ee4d0 0xc0002ee5b0 0xc0002ee620 0xc000be2a00 0xc0002ee690 0xc0002ee700 0xc000aa0370 0xc0065b9570 0xc0002ee770 map[0x2ff9080:0xc002a93da0 0x2ff95c0:0xc002a93f20 0x2ffa2e0:0xc002616e40 0x2ffa580:0xc002a93d40 0x2ffad60:0xc002a93e00 0x2ffb000:0xc002a93e60 0x2ffb2a0:0xc002a93ec0 0x2ffba80:0xc0066ca000 0x2ffbd20:0xc0066ca060 0x3026980:0xc001707860 0x3026d20:0xc001707a40 0x3027800:0xc000bbde60 0x3029160:0xc001c0e000 0x30298a0:0xc001707aa0 0x302a720:0xc0010163c0] 0xc0006beee0 0} I0423 19:58:12.818588 3643 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc2490 0xc0002ee380 0xc0002ee3f0 0xc0002ee460 0xc0002ee4d0 0xc0002ee5b0 0xc0002ee620 0xc000be2a00 0xc0002ee690 0xc0002ee700 0xc000aa0370 0xc0065b9570 0xc0002ee770 map[0x2ff9080:0xc002a93da0 0x2ff95c0:0xc002a93f20 0x2ffa2e0:0xc002616e40 0x2ffa580:0xc002a93d40 0x2ffad60:0xc002a93e00 0x2ffb000:0xc002a93e60 0x2ffb2a0:0xc002a93ec0 0x2ffba80:0xc0066ca000 0x2ffbd20:0xc0066ca060 0x3026980:0xc001707860 0x3026d20:0xc001707a40 0x3027800:0xc000bbde60 0x3029160:0xc001c0e000 0x30298a0:0xc001707aa0 0x302a720:0xc0010163c0] 0xc0006beee0 0} I0423 19:58:12.818632 3643 gateway.go:532] Reconciling gateway with updates I0423 19:58:12.823867 3643 gateway.go:339] Spawning Conntrack Rule Check Thread I0423 19:58:12.823955 3643 default_node_network_controller.go:944] Gateway and management port readiness took 62.560166ms I0423 19:58:12.824025 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:58:12.824045 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:58:12.824038 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:58:12.824059 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:58:12.824060 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:58:12.824066 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:58:12.824076 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:58:12.824079 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:58:12.824084 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 19:58:12.824088 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 19:58:12.824088 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 19:58:12.824052 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:58:12.824093 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:58:12.824105 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 19:58:12.824106 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:58:12.824109 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:58:12.824122 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:58:12.824125 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:58:12.824131 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:58:12.824129 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 19:58:12.824138 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:58:12.824139 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 19:58:12.824132 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:58:12.824069 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:58:12.824145 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:58:12.824148 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:58:12.824155 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 19:58:12.824158 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:58:12.824162 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:58:12.824165 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:58:12.824176 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:58:12.824122 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 19:58:12.824180 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:58:12.824186 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 19:58:12.824186 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 19:58:12.824192 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 19:58:12.824195 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:58:12.824185 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:58:12.824201 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 19:58:12.824207 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 32086 is already open, no action needed I0423 19:58:12.824206 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:58:12.824212 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 19:58:12.824214 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 19:58:12.824111 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:58:12.824218 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:58:12.824233 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:58:12.824235 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 19:58:12.824235 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:58:12.824203 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 19:58:12.824243 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 19:58:12.824249 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:58:12.824254 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:58:12.824252 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:58:12.824260 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:58:12.824251 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:58:12.824264 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:58:12.824264 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:58:12.824257 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:58:12.824289 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:58:12.824291 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 19:58:12.824300 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:58:12.824294 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:58:12.824305 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:58:12.824316 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:58:12.824316 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 19:58:12.824323 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:58:12.824233 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 19:58:12.824331 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:58:12.824330 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:58:12.824335 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 19:58:12.824339 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 19:58:12.824347 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:58:12.824318 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 19:58:12.824220 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 32481 is already open, no action needed I0423 19:58:12.824100 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:58:12.824382 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:58:12.824025 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:58:12.824409 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:58:12.824240 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:58:12.824414 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 19:58:12.824419 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:58:12.824428 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 19:58:12.824435 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:58:12.824448 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:58:12.824258 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:58:12.824463 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:58:12.824339 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 19:58:12.824196 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:58:12.824483 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:58:12.824512 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:58:12.824526 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:58:12.824547 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 19:58:12.824344 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 19:58:12.824554 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:58:12.824203 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 19:58:12.824353 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 19:58:12.824561 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:58:12.824563 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:58:12.824573 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:58:12.824574 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:58:12.824576 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 19:58:12.824582 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:58:12.824586 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 19:58:12.824584 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 19:58:12.824585 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:58:12.824600 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:58:12.824598 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 19:58:12.824615 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 19:58:12.824562 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 19:58:12.824669 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 19:58:12.824676 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 19:58:12.824236 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:58:12.824686 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:58:12.824417 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:58:12.825168 3643 node_ip_handler_linux.go:160] Node IP manager is running W0423 19:58:12.863969 3643 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 19:58:12.864239 3643 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 19:58:12.864431 3643 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0423 19:58:12.871836 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 19:58:12.871864 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 19:58:12.871872 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 19:58:12.871880 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 19:58:12.912295 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 19:58:12.912309 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 19:58:12.912298 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 19:58:12.912325 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 19:58:12.912331 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 19:58:12.912336 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 19:58:12.912343 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 19:58:12.912348 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 19:58:12.912365 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 19:58:12.912379 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 19:58:12.952252 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 19:58:12.952299 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 19:58:12.952315 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 19:58:12.952322 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 19:58:12.952328 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 19:58:12.952334 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 19:58:13.109133 3643 default_network_controller.go:1025] Node "ip-10-0-0-111.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:58:14.164817 3643 default_node_network_controller.go:1380] Starting node controller node sync W0423 19:58:14.164883 3643 egressip_healthcheck.go:76] Health checking using insecure connection I0423 19:58:14.164995 3643 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0423 19:58:14.176327 3643 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.488929ms I0423 19:58:14.176491 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-102.ec2.internal", IP: 10.0.2.102 to PMTUD blocking rules I0423 19:58:14.176502 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-102.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:58:14.181812 3643 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0423 19:58:14.182083 3643 egressservice_node.go:93] Setting up event handlers for Egress Services I0423 19:58:14.182193 3643 egressservice_node.go:183] Starting Egress Services Controller I0423 19:58:14.182208 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 19:58:14.182217 3643 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 19:58:14.182224 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 19:58:14.182229 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 19:58:14.182234 3643 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 19:58:14.182239 3643 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 19:58:14.182243 3643 egressservice_node.go:197] Repairing Egress Services I0423 19:58:14.192501 3643 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0423 19:58:14.192515 3643 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 19:58:14.192704 3643 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0423 19:58:14.192718 3643 link_network_manager.go:119] Link manager is running I0423 19:58:14.192727 3643 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0423 19:58:14.192979 3643 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0423 19:58:14.193493 3643 vrf_manager.go:128] VRF manager is running I0423 19:58:14.193767 3643 node_controller_manager.go:437] Removing flows to drop GARP I0423 19:58:14.193776 3643 gateway.go:532] Reconciling gateway with updates I0423 19:58:14.221231 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:58:14.221254 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:58:14.221262 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 19:58:14.221262 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 19:58:14.221288 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 19:58:14.221296 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:58:14.221298 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 19:58:14.221312 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 19:58:14.221313 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:58:14.221321 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 19:58:14.221333 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:58:14.221334 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:58:14.221338 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 19:58:14.221241 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:58:14.221347 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 19:58:14.221350 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 19:58:14.221357 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 19:58:14.221360 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 19:58:14.221362 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 19:58:14.221369 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 19:58:14.221373 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 19:58:14.221392 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 19:58:14.221393 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 19:58:14.221410 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 19:58:14.221420 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:58:14.221434 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 19:58:14.221444 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 19:58:14.221370 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 19:58:14.221492 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:58:14.221495 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:58:14.221508 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 19:58:14.221507 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:58:14.221513 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:58:14.221522 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 19:58:14.221523 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 19:58:14.221529 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:58:14.221531 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 19:58:14.221533 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:58:14.221538 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:58:14.221550 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 19:58:14.221551 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 19:58:14.221555 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:58:14.221561 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 19:58:14.221570 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 19:58:14.221573 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:58:14.221542 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 19:58:14.221579 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 19:58:14.221578 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 19:58:14.221586 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 19:58:14.221593 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 19:58:14.221595 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:58:14.221599 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:58:14.221613 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 19:58:14.221619 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 19:58:14.221621 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 19:58:14.221623 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:58:14.221625 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 19:58:14.221635 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 19:58:14.221604 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:58:14.221674 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 19:58:14.221494 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:58:14.221681 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:58:14.221685 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:58:14.221692 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:58:14.221691 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:58:14.221702 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 19:58:14.221701 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 19:58:14.221707 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 19:58:14.221713 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 19:58:14.221720 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:58:14.221686 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:58:14.221734 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 19:58:14.221742 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:58:14.221754 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:58:14.221758 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:58:14.221754 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:58:14.221760 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 19:58:14.221767 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:58:14.221769 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:58:14.221770 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:58:14.221775 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 19:58:14.221773 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 19:58:14.221782 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 19:58:14.221783 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:58:14.221792 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 32086 is already open, no action needed I0423 19:58:14.221795 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 19:58:14.221800 3643 port_claim.go:71] Svc openshift-ingress/router-default: port 32481 is already open, no action needed I0423 19:58:14.221796 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 19:58:14.221811 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 19:58:14.221698 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 19:58:14.221743 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 19:58:14.221698 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 19:58:14.221841 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 19:58:14.221686 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:58:14.221865 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 19:58:14.221767 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 19:58:14.221761 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 19:58:14.221735 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 19:58:14.221692 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 19:58:14.221966 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 19:58:14.221971 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 19:58:14.221747 3643 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:58:14.221979 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 19:58:14.221743 3643 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 19:58:14.261692 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 19:58:14.261721 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 19:58:14.261726 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 19:58:14.261734 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 19:58:14.261743 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 19:58:14.261751 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 19:58:14.261760 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 19:58:14.261779 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 19:58:14.301314 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 19:58:14.301339 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301345 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301351 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 19:58:14.301357 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 19:58:14.301364 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301370 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 19:58:14.301375 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301380 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301386 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301392 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 19:58:14.301397 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 19:58:14.301402 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 19:58:14.301408 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 19:58:14.340966 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 19:58:14.340988 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 19:58:14.340996 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 19:58:14.341002 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 19:58:14.341007 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 19:58:14.341014 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 19:58:14.341019 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 19:58:14.341024 3643 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) W0423 19:58:14.393160 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 19:58:16.131926 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-149.ec2.internal I0423 19:58:16.131955 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-149.ec2.internal: 36.508µs I0423 19:58:16.165375 3643 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6594b6fc67-z8lh9] creating logical port openshift-image-registry_image-registry-6594b6fc67-z8lh9 for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:16.165512 3643 kube.go:256] Updating pod openshift-image-registry/image-registry-6594b6fc67-z8lh9 I0423 19:58:16.165521 3643 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7786db44fb-jd77k] creating logical port openshift-ingress_router-default-7786db44fb-jd77k for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:16.165740 3643 kube.go:256] Updating pod openshift-ingress/router-default-7786db44fb-jd77k I0423 19:58:16.195449 3643 pod.go:62] [openshift-image-registry/image-registry-6594b6fc67-z8lh9] pod update took 29.966024ms I0423 19:58:16.195472 3643 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6594b6fc67-z8lh9] addLogicalPort annotation time took 29.996331ms I0423 19:58:16.195494 3643 pod.go:62] [openshift-ingress/router-default-7786db44fb-jd77k] pod update took 29.791075ms I0423 19:58:16.195509 3643 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7786db44fb-jd77k] addLogicalPort annotation time took 29.819345ms I0423 19:58:16.195824 3643 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-46d5m] creating logical port openshift-ingress-canary_ingress-canary-46d5m for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:16.195958 3643 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-4ktbq] creating logical port openshift-dns_dns-default-4ktbq for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:16.195967 3643 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-46d5m I0423 19:58:16.196042 3643 kube.go:256] Updating pod openshift-dns/dns-default-4ktbq I0423 19:58:16.196904 3643 pods.go:271] [openshift-image-registry/image-registry-6594b6fc67-z8lh9] addLogicalPort took 31.545531ms, libovsdb time 1.110038ms I0423 19:58:16.197461 3643 pods.go:271] [openshift-ingress/router-default-7786db44fb-jd77k] addLogicalPort took 31.950542ms, libovsdb time 1.620951ms I0423 19:58:16.211001 3643 pod.go:62] [openshift-ingress-canary/ingress-canary-46d5m] pod update took 15.088826ms I0423 19:58:16.211020 3643 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-46d5m] addLogicalPort annotation time took 15.111273ms I0423 19:58:16.212120 3643 pods.go:271] [openshift-ingress-canary/ingress-canary-46d5m] addLogicalPort took 16.309425ms, libovsdb time 789.153µs I0423 19:58:16.214519 3643 pod.go:62] [openshift-dns/dns-default-4ktbq] pod update took 18.497793ms I0423 19:58:16.214539 3643 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-4ktbq] addLogicalPort annotation time took 18.518152ms I0423 19:58:16.215554 3643 pods.go:271] [openshift-dns/dns-default-4ktbq] addLogicalPort took 19.60759ms, libovsdb time 785.995µs I0423 19:58:16.597719 3643 cni.go:452] [openshift-ingress-canary/ingress-canary-46d5m 28dcbf5abd9aec26759c78ccd988ae714ab551403298762307d531034cd8e5c3 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-46d5m 28dcbf5abd9aec26759c78ccd988ae714ab551403298762307d531034cd8e5c3 network default NAD default NAD key ] I0423 19:58:16.642242 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-46d5m, hostIfaceName: 28dcbf5abd9aec2, network: default, NAD default, SandboxID: "28dcbf5abd9aec26759c78ccd988ae714ab551403298762307d531034cd8e5c3", PCI device ID: , UID: "c6923ac6-e48e-47ec-9ea1-2bfbc4dcb378", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0423 19:58:16.716154 3643 cni.go:473] [openshift-ingress-canary/ingress-canary-46d5m 28dcbf5abd9aec26759c78ccd988ae714ab551403298762307d531034cd8e5c3 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-46d5m 28dcbf5abd9aec26759c78ccd988ae714ab551403298762307d531034cd8e5c3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:d0:f6:ab:d1:f8\",\"name\":\"28dcbf5abd9aec2\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/99f51bd1-3d52-4584-b42f-1c75303879e2\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:19.310381 3643 default_network_controller.go:1025] Node "ip-10-0-0-111.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:58:20.260882 3643 cni.go:452] [openshift-network-diagnostics/network-check-target-xswps a25984eadea8585c105b633d8ecd66edf3ed59d434a3766a6be21c98f25d8408 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-xswps a25984eadea8585c105b633d8ecd66edf3ed59d434a3766a6be21c98f25d8408 network default NAD default NAD key ] I0423 19:58:20.271006 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-xswps, hostIfaceName: a25984eadea8585, network: default, NAD default, SandboxID: "a25984eadea8585c105b633d8ecd66edf3ed59d434a3766a6be21c98f25d8408", PCI device ID: , UID: "037566c0-528f-4f6c-a1f2-00fd56571629", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0423 19:58:20.319117 3643 cni.go:473] [openshift-network-diagnostics/network-check-target-xswps a25984eadea8585c105b633d8ecd66edf3ed59d434a3766a6be21c98f25d8408 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-xswps a25984eadea8585c105b633d8ecd66edf3ed59d434a3766a6be21c98f25d8408 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:b3:a6:fd:de:e5\",\"name\":\"a25984eadea8585\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8125a934-0264-4e06-974c-025fcc99381a\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:20.379539 3643 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0423 19:58:20.380456 3643 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 892.749µs I0423 19:58:20.433182 3643 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 19:58:20.672776 3643 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-v8fhz for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:20.672897 3643 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz I0423 19:58:20.697413 3643 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz] pod update took 24.540431ms I0423 19:58:20.697441 3643 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz] addLogicalPort annotation time took 24.571532ms I0423 19:58:20.698817 3643 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz] addLogicalPort took 26.060964ms, libovsdb time 1.097056ms W0423 19:58:20.779352 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:58:21.132168 3643 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz f16c26e872d79a9f1d40e26df7b060eba4f32112ee1caf0d5f3f580e107d3984 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz f16c26e872d79a9f1d40e26df7b060eba4f32112ee1caf0d5f3f580e107d3984 network default NAD default NAD key ] I0423 19:58:21.142004 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-v8fhz, hostIfaceName: f16c26e872d79a9, network: default, NAD default, SandboxID: "f16c26e872d79a9f1d40e26df7b060eba4f32112ee1caf0d5f3f580e107d3984", PCI device ID: , UID: "771e08a3-c1ca-4b5b-aab4-e03c5f5f51d9", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0423 19:58:21.190256 3643 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz f16c26e872d79a9f1d40e26df7b060eba4f32112ee1caf0d5f3f580e107d3984 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-v8fhz f16c26e872d79a9f1d40e26df7b060eba4f32112ee1caf0d5f3f580e107d3984 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:7b:a5:b4:82:47\",\"name\":\"f16c26e872d79a9\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8eb9648d-cf6e-42c2-a157-05ba7f998cf7\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:21.853662 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-111.ec2.internal I0423 19:58:21.853684 3643 default_network_controller.go:1025] Node "ip-10-0-0-111.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:58:21.853688 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-111.ec2.internal: 32.743µs I0423 19:58:21.853688 3643 egressfirewall.go:723] Syncing node "ip-10-0-0-111.ec2.internal" for egress firewall I0423 19:58:21.936583 3643 namespace.go:98] [openshift-service-ca] adding namespace I0423 19:58:21.937541 3643 namespace.go:102] [openshift-service-ca] adding namespace took 932.105µs I0423 19:58:21.951888 3643 namespace.go:142] [openshift-service-ca] updating namespace I0423 19:58:23.627127 3643 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-n9f5c] creating logical port openshift-service-ca_service-ca-d9cb48497-n9f5c for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:23.627233 3643 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-n9f5c I0423 19:58:23.640250 3643 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-n9f5c] pod update took 13.041765ms I0423 19:58:23.640294 3643 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-n9f5c] addLogicalPort annotation time took 13.088369ms I0423 19:58:23.641549 3643 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-n9f5c] addLogicalPort took 14.440479ms, libovsdb time 973.818µs I0423 19:58:24.005941 3643 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-n9f5c 088e3238b93a419699f427c0c10b6ee89dacb08ffff49479e4961c3824a3683b network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-n9f5c 088e3238b93a419699f427c0c10b6ee89dacb08ffff49479e4961c3824a3683b network default NAD default NAD key ] I0423 19:58:24.017225 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-n9f5c, hostIfaceName: 088e3238b93a419, network: default, NAD default, SandboxID: "088e3238b93a419699f427c0c10b6ee89dacb08ffff49479e4961c3824a3683b", PCI device ID: , UID: "ca2c058a-7cf9-4aa3-9d07-1148d6d18dcf", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0423 19:58:24.066175 3643 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-n9f5c 088e3238b93a419699f427c0c10b6ee89dacb08ffff49479e4961c3824a3683b network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-n9f5c 088e3238b93a419699f427c0c10b6ee89dacb08ffff49479e4961c3824a3683b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:58:67:18:79:14\",\"name\":\"088e3238b93a419\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f8f63e4e-72b9-4c7b-9deb-73bfd0f7641e\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:58:24.480834 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 19:58:27.637705 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-111.ec2.internal in network "default" I0423 19:58:27.637759 3643 node_tracker.go:235] Node ip-10-0-0-111.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-111.ec2.internal" I0423 19:58:27.637783 3643 node_tracker.go:169] Node ip-10-0-0-111.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:27.637708 3643 default_network_controller.go:1025] Node "ip-10-0-0-111.ec2.internal" in remote zone "ip-10-0-0-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:58:27.637802 3643 services_controller.go:574] Full service sync requested for network=default W0423 19:58:27.638211 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:58:27.638218 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 19:58:27.638327 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 19:58:27.682523 3643 default_network_controller.go:1025] Node "ip-10-0-0-111.ec2.internal" in remote zone "ip-10-0-0-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:58:27.798316 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-111.ec2.internal in network "default" I0423 19:58:27.798320 3643 default_network_controller.go:1025] Node "ip-10-0-0-111.ec2.internal" in remote zone "ip-10-0-0-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 19:58:27.798368 3643 node_tracker.go:169] Node ip-10-0-0-111.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:27.798378 3643 services_controller.go:574] Full service sync requested for network=default W0423 19:58:27.798678 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:58:27.798683 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0423 19:58:27.798750 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 19:58:27.799294 3643 zone_ic_handler.go:195] Time taken to create transit switch: 268.497µs I0423 19:58:27.799308 3643 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-111.ec2.internal for the network default I0423 19:58:27.800300 3643 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-111.ec2.internal" on network "default" took: 1.283359ms I0423 19:58:28.852796 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-111.ec2.internal I0423 19:58:28.852818 3643 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-111.ec2.internal in network "default" I0423 19:58:28.852824 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-111.ec2.internal: 32.79µs I0423 19:58:28.852798 3643 egressfirewall.go:723] Syncing node "ip-10-0-0-111.ec2.internal" for egress firewall I0423 19:58:28.852871 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-111.ec2.internal", IP: 10.0.0.111 to PMTUD blocking rules I0423 19:58:28.852884 3643 node_tracker.go:169] Node ip-10-0-0-111.ec2.internal switch + router changed, syncing services in network "default" I0423 19:58:28.852885 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-111.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 19:58:28.852900 3643 services_controller.go:574] Full service sync requested for network=default W0423 19:58:28.853047 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 19:58:28.853116 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:58:28.853053 3643 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 19:58:28.859168 3643 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-111.ec2.internal. Updating no re-route policies I0423 19:58:28.871427 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-111.ec2.internal", IP: 10.0.0.111 to PMTUD blocking rules I0423 19:58:28.871442 3643 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-111.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0423 19:58:31.536735 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 19:58:32.160294 3643 cni.go:452] [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key ] I0423 19:58:32.171899 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6594b6fc67-z8lh9, hostIfaceName: f94837ce702c406, network: default, NAD default, SandboxID: "f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84", PCI device ID: , UID: "03090210-ce73-4156-b679-0dbc38e89d3e", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0423 19:58:32.218959 3643 cni.go:452] [openshift-dns/dns-default-4ktbq ca0d7e9bf1e5fd087661f879834b9417aae3dd20e806bbb05e487fd27bf9cb7c network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-4ktbq ca0d7e9bf1e5fd087661f879834b9417aae3dd20e806bbb05e487fd27bf9cb7c network default NAD default NAD key ] I0423 19:58:32.225115 3643 cni.go:473] [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:9f:96:5c:08:8d\",\"name\":\"f94837ce702c406\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a6f928f5-04d9-4673-a895-e5ff9299fec3\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:32.231418 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-4ktbq, hostIfaceName: ca0d7e9bf1e5fd0, network: default, NAD default, SandboxID: "ca0d7e9bf1e5fd087661f879834b9417aae3dd20e806bbb05e487fd27bf9cb7c", PCI device ID: , UID: "d323130a-7870-481d-9752-063712f992a2", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0423 19:58:32.278043 3643 cni.go:473] [openshift-dns/dns-default-4ktbq ca0d7e9bf1e5fd087661f879834b9417aae3dd20e806bbb05e487fd27bf9cb7c network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-4ktbq ca0d7e9bf1e5fd087661f879834b9417aae3dd20e806bbb05e487fd27bf9cb7c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:4c:e8:4b:7c:ad\",\"name\":\"ca0d7e9bf1e5fd0\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e4b4a039-dc0b-43a6-9113-cd59aad45663\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:58:47.227848 3643 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:58:47.241376 3643 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 19:58:47.337205 3643 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-cllgq] creating logical port openshift-insights_insights-runtime-extractor-cllgq for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:47.337341 3643 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-cllgq I0423 19:58:47.364250 3643 pod.go:62] [openshift-insights/insights-runtime-extractor-cllgq] pod update took 26.947068ms I0423 19:58:47.364294 3643 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-cllgq] addLogicalPort annotation time took 26.993346ms I0423 19:58:47.365423 3643 pods.go:271] [openshift-insights/insights-runtime-extractor-cllgq] addLogicalPort took 28.234777ms, libovsdb time 858.297µs W0423 19:58:47.371778 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-785b47677b-wczkv IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-785b47677b-wczkv: no pod IPs found W0423 19:58:47.399833 3643 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:58:47.441500 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-6x6q8 for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:47.441583 3643 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8 I0423 19:58:47.442669 3643 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-n9gzv] creating logical port openshift-console_downloads-5c54f4944f-n9gzv for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:47.442761 3643 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-n9gzv I0423 19:58:47.464875 3643 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8] pod update took 23.314919ms I0423 19:58:47.464895 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8] addLogicalPort annotation time took 23.340628ms I0423 19:58:47.466058 3643 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8] addLogicalPort took 24.568113ms, libovsdb time 863.091µs I0423 19:58:47.468488 3643 pod.go:62] [openshift-console/downloads-5c54f4944f-n9gzv] pod update took 25.755652ms I0423 19:58:47.468507 3643 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-n9gzv] addLogicalPort annotation time took 25.779358ms I0423 19:58:47.469895 3643 pods.go:271] [openshift-console/downloads-5c54f4944f-n9gzv] addLogicalPort took 27.251349ms, libovsdb time 1.004807ms I0423 19:58:47.630680 3643 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-111.ec2.internal I0423 19:58:47.630709 3643 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-111.ec2.internal: 38.277µs I0423 19:58:47.703036 3643 cni.go:452] [openshift-insights/insights-runtime-extractor-cllgq 274a92a7ae1fe1b576b0594207d56373c9b631fd90102263867fca9367101410 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-cllgq 274a92a7ae1fe1b576b0594207d56373c9b631fd90102263867fca9367101410 network default NAD default NAD key ] I0423 19:58:47.712248 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-cllgq, hostIfaceName: 274a92a7ae1fe1b, network: default, NAD default, SandboxID: "274a92a7ae1fe1b576b0594207d56373c9b631fd90102263867fca9367101410", PCI device ID: , UID: "1ba5b8fe-48b7-419f-9b85-8c77bb800dff", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0423 19:58:47.760661 3643 cni.go:473] [openshift-insights/insights-runtime-extractor-cllgq 274a92a7ae1fe1b576b0594207d56373c9b631fd90102263867fca9367101410 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-cllgq 274a92a7ae1fe1b576b0594207d56373c9b631fd90102263867fca9367101410 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:fc:7d:e9:13:d2\",\"name\":\"274a92a7ae1fe1b\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a0f7417-ca59-4391-9b41-7fdf2a58a462\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:47.800911 3643 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8 be5ca2f143fa7e5b1c9b4465d8bf57842075b48ba26eb522863deda3bc192ab6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8 be5ca2f143fa7e5b1c9b4465d8bf57842075b48ba26eb522863deda3bc192ab6 network default NAD default NAD key ] I0423 19:58:47.808775 3643 cni.go:452] [openshift-console/downloads-5c54f4944f-n9gzv 7316fd14b07f705b6f7206166c151f630cdb49a7221e09bbde85b1fcd0540985 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-n9gzv 7316fd14b07f705b6f7206166c151f630cdb49a7221e09bbde85b1fcd0540985 network default NAD default NAD key ] I0423 19:58:47.812835 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-6x6q8, hostIfaceName: be5ca2f143fa7e5, network: default, NAD default, SandboxID: "be5ca2f143fa7e5b1c9b4465d8bf57842075b48ba26eb522863deda3bc192ab6", PCI device ID: , UID: "87103699-49be-4770-b99c-364ee2356db7", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0423 19:58:47.820571 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-n9gzv, hostIfaceName: 7316fd14b07f705, network: default, NAD default, SandboxID: "7316fd14b07f705b6f7206166c151f630cdb49a7221e09bbde85b1fcd0540985", PCI device ID: , UID: "1f7968c0-415c-4f38-a265-2815e6797b81", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0423 19:58:47.863585 3643 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8 be5ca2f143fa7e5b1c9b4465d8bf57842075b48ba26eb522863deda3bc192ab6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-6x6q8 be5ca2f143fa7e5b1c9b4465d8bf57842075b48ba26eb522863deda3bc192ab6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:fa:3b:54:60:dc\",\"name\":\"be5ca2f143fa7e5\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ca1e699-fa84-4304-8443-4327af9ec15f\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:47.882292 3643 cni.go:473] [openshift-console/downloads-5c54f4944f-n9gzv 7316fd14b07f705b6f7206166c151f630cdb49a7221e09bbde85b1fcd0540985 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-n9gzv 7316fd14b07f705b6f7206166c151f630cdb49a7221e09bbde85b1fcd0540985 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:41:3a:75:ac:d1\",\"name\":\"7316fd14b07f705\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3669f70-90e2-40df-bae3-f79cd449bfa9\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:48.360648 3643 cni.go:452] [openshift-ingress/router-default-7786db44fb-jd77k bf0f47be646aa27e3d07e279e2b81aaa8b0129b7569c8d6664bb4015dc676c74 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7786db44fb-jd77k bf0f47be646aa27e3d07e279e2b81aaa8b0129b7569c8d6664bb4015dc676c74 network default NAD default NAD key ] I0423 19:58:48.371633 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7786db44fb-jd77k, hostIfaceName: bf0f47be646aa27, network: default, NAD default, SandboxID: "bf0f47be646aa27e3d07e279e2b81aaa8b0129b7569c8d6664bb4015dc676c74", PCI device ID: , UID: "ad6d07d2-1492-4bf8-8ce6-2e29f6eb9051", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0423 19:58:48.428237 3643 cni.go:473] [openshift-ingress/router-default-7786db44fb-jd77k bf0f47be646aa27e3d07e279e2b81aaa8b0129b7569c8d6664bb4015dc676c74 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7786db44fb-jd77k bf0f47be646aa27e3d07e279e2b81aaa8b0129b7569c8d6664bb4015dc676c74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:61:fc:be:72:cc\",\"name\":\"bf0f47be646aa27\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ec78aaa7-4ccb-4a7d-a52a-1b651672a8d7\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:58:49.023642 3643 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 19:58:49.517080 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0423 19:58:49.537712 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0423 19:58:49.537734 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0423 19:58:49.560242 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0423 19:58:50.572934 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0423 19:58:50.572970 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0423 19:58:50.649288 3643 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0423 19:58:50.663005 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:58:50.663024 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 19:58:50.663029 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0423 19:58:51.480882 3643 base_network_controller_pods.go:487] [default/openshift-console/console-d74f58697-25ltm] creating logical port openshift-console_console-d74f58697-25ltm for pod on switch ip-10-0-1-149.ec2.internal I0423 19:58:51.481037 3643 kube.go:256] Updating pod openshift-console/console-d74f58697-25ltm I0423 19:58:51.498146 3643 pod.go:62] [openshift-console/console-d74f58697-25ltm] pod update took 17.136946ms I0423 19:58:51.498177 3643 base_network_controller_pods.go:951] [default/openshift-console/console-d74f58697-25ltm] addLogicalPort annotation time took 17.174482ms I0423 19:58:51.499532 3643 pods.go:271] [openshift-console/console-d74f58697-25ltm] addLogicalPort took 18.665245ms, libovsdb time 944.901µs I0423 19:58:51.878790 3643 cni.go:452] [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key ] I0423 19:58:51.888138 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-d74f58697-25ltm, hostIfaceName: 5312bf4826e3386, network: default, NAD default, SandboxID: "5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b", PCI device ID: , UID: "0056b6c7-0f5d-4975-b358-f5f7a678daea", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0423 19:58:51.939922 3643 cni.go:473] [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:15:81:f2:4a:c5\",\"name\":\"5312bf4826e3386\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c7ee788-d406-4db4-b96b-6a9abec81062\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:58:52.336257 3643 cni.go:452] [openshift-multus/network-metrics-daemon-cgz9p 2186b23b20487020fd6c0575c8684b0c0878b8058d4a1887ddab44f9e188b3a8 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-cgz9p 2186b23b20487020fd6c0575c8684b0c0878b8058d4a1887ddab44f9e188b3a8 network default NAD default NAD key ] I0423 19:58:52.345765 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-cgz9p, hostIfaceName: 2186b23b2048702, network: default, NAD default, SandboxID: "2186b23b20487020fd6c0575c8684b0c0878b8058d4a1887ddab44f9e188b3a8", PCI device ID: , UID: "5f3ded06-6f10-45ad-b127-fc7223c85fca", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0423 19:58:52.396656 3643 cni.go:473] [openshift-multus/network-metrics-daemon-cgz9p 2186b23b20487020fd6c0575c8684b0c0878b8058d4a1887ddab44f9e188b3a8 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-cgz9p 2186b23b20487020fd6c0575c8684b0c0878b8058d4a1887ddab44f9e188b3a8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:fe:51:58:87:46\",\"name\":\"2186b23b2048702\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c1af53d-ce8b-4075-99d4-bae57e19c5cb\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:58:55.436644 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 19:58:56.070881 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:58:56.555450 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 19:58:57.420635 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:59:00.366458 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-149.ec2.internal I0423 19:59:00.366601 3643 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0423 19:59:00.386851 3643 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 20.283267ms I0423 19:59:00.386882 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 20.319809ms I0423 19:59:00.388337 3643 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 21.894191ms, libovsdb time 1.078476ms W0423 19:59:00.514158 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 19:59:01.400243 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-555c97f49f-c6xgb] creating logical port openshift-monitoring_thanos-querier-555c97f49f-c6xgb for pod on switch ip-10-0-1-149.ec2.internal I0423 19:59:01.400408 3643 kube.go:256] Updating pod openshift-monitoring/thanos-querier-555c97f49f-c6xgb I0423 19:59:01.451321 3643 kube.go:256] Updating pod openshift-monitoring/thanos-querier-555c97f49f-c6xgb I0423 19:59:01.471729 3643 pod.go:62] [openshift-monitoring/thanos-querier-555c97f49f-c6xgb] pod update took 71.367026ms I0423 19:59:01.471760 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-555c97f49f-c6xgb] addLogicalPort annotation time took 71.404677ms I0423 19:59:01.473241 3643 pods.go:271] [openshift-monitoring/thanos-querier-555c97f49f-c6xgb] addLogicalPort took 73.014109ms, libovsdb time 985.805µs W0423 19:59:02.584896 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 19:59:03.125146 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:03.125165 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:03.125176 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 19:59:03.889554 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-774cd65c5-6spt2] creating logical port openshift-monitoring_metrics-server-774cd65c5-6spt2 for pod on switch ip-10-0-1-149.ec2.internal I0423 19:59:03.889705 3643 kube.go:256] Updating pod openshift-monitoring/metrics-server-774cd65c5-6spt2 I0423 19:59:03.908622 3643 pod.go:62] [openshift-monitoring/metrics-server-774cd65c5-6spt2] pod update took 18.940881ms I0423 19:59:03.908654 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-774cd65c5-6spt2] addLogicalPort annotation time took 18.977995ms I0423 19:59:03.909840 3643 pods.go:271] [openshift-monitoring/metrics-server-774cd65c5-6spt2] addLogicalPort took 20.309413ms, libovsdb time 843.044µs I0423 19:59:04.014390 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-gczfd for pod on switch ip-10-0-1-149.ec2.internal I0423 19:59:04.014501 3643 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd I0423 19:59:04.030720 3643 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd] pod update took 16.254804ms I0423 19:59:04.030745 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd] addLogicalPort annotation time took 16.283774ms I0423 19:59:04.032087 3643 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd] addLogicalPort took 17.715866ms, libovsdb time 981.301µs W0423 19:59:05.688546 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-6bcc7cdd95-4r95f IP address from the namespace address-set, err: pod openshift-console/console-6bcc7cdd95-4r95f: no pod IPs found W0423 19:59:07.125620 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:07.125639 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:07.125645 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:07.125649 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:07.167490 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:59:07.167510 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:59:07.214093 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0423 19:59:07.217099 3643 cni.go:452] [openshift-monitoring/metrics-server-774cd65c5-6spt2 bd5b9510b043e289aa56b420e253d63278150da3230cc4f88c168577a2e6b0ae network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-774cd65c5-6spt2 bd5b9510b043e289aa56b420e253d63278150da3230cc4f88c168577a2e6b0ae network default NAD default NAD key ] I0423 19:59:07.221048 3643 cni.go:452] [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key ] I0423 19:59:07.221938 3643 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd c1e225d9177fcac67cd4484266cf6f7824e6d10369f62ebe843c46a2898ae1ea network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd c1e225d9177fcac67cd4484266cf6f7824e6d10369f62ebe843c46a2898ae1ea network default NAD default NAD key ] I0423 19:59:07.223524 3643 cni.go:452] [openshift-monitoring/thanos-querier-555c97f49f-c6xgb 5ad0c718e67d8401f7cba75b0e61081f89315bd3aa49b27dba925a9f865b8ea1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-555c97f49f-c6xgb 5ad0c718e67d8401f7cba75b0e61081f89315bd3aa49b27dba925a9f865b8ea1 network default NAD default NAD key ] I0423 19:59:07.228574 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-774cd65c5-6spt2, hostIfaceName: bd5b9510b043e28, network: default, NAD default, SandboxID: "bd5b9510b043e289aa56b420e253d63278150da3230cc4f88c168577a2e6b0ae", PCI device ID: , UID: "ff6820ba-2e86-49a2-9296-8f52a8118678", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0423 19:59:07.235704 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-555c97f49f-c6xgb, hostIfaceName: 5ad0c718e67d840, network: default, NAD default, SandboxID: "5ad0c718e67d8401f7cba75b0e61081f89315bd3aa49b27dba925a9f865b8ea1", PCI device ID: , UID: "f3c5876c-93bb-4bbf-8347-503ada2cd4fd", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0423 19:59:07.238830 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-gczfd, hostIfaceName: c1e225d9177fcac, network: default, NAD default, SandboxID: "c1e225d9177fcac67cd4484266cf6f7824e6d10369f62ebe843c46a2898ae1ea", PCI device ID: , UID: "f107ae0d-6401-4fdd-82a3-b0af958354cc", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0423 19:59:07.246911 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 7418cc66df3bd09, network: default, NAD default, SandboxID: "7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68", PCI device ID: , UID: "d9b56c5b-92d7-4927-8fb9-cc29fab23522", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0423 19:59:07.292867 3643 cni.go:473] [openshift-monitoring/metrics-server-774cd65c5-6spt2 bd5b9510b043e289aa56b420e253d63278150da3230cc4f88c168577a2e6b0ae network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-774cd65c5-6spt2 bd5b9510b043e289aa56b420e253d63278150da3230cc4f88c168577a2e6b0ae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:b8:93:1d:3e:f0\",\"name\":\"bd5b9510b043e28\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13e81958-71a5-4c81-8aa8-68dbc742d1bb\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:59:07.517071 3643 cni.go:473] [openshift-monitoring/thanos-querier-555c97f49f-c6xgb 5ad0c718e67d8401f7cba75b0e61081f89315bd3aa49b27dba925a9f865b8ea1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-555c97f49f-c6xgb 5ad0c718e67d8401f7cba75b0e61081f89315bd3aa49b27dba925a9f865b8ea1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:30:52:97:11:c2\",\"name\":\"5ad0c718e67d840\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de86bb12-d8df-44d6-846a-3e0d927160c1\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:59:07.518518 3643 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd c1e225d9177fcac67cd4484266cf6f7824e6d10369f62ebe843c46a2898ae1ea network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-gczfd c1e225d9177fcac67cd4484266cf6f7824e6d10369f62ebe843c46a2898ae1ea network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:81:42:b7:1e:14\",\"name\":\"c1e225d9177fcac\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f03ef4dc-afbd-4c58-a621-8993842a32f6\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 19:59:07.518570 3643 cni.go:473] [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:9a:31:94:27:6f\",\"name\":\"7418cc66df3bd09\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55a65c75-2787-4240-8cc5-0a09bd2717eb\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:59:08.701453 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:59:08.701505 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 19:59:10.649756 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:10.649782 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:10.649789 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:10.676966 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:59:10.701025 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0423 19:59:11.637126 3643 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:11.637146 3643 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:11.637152 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:11.637156 3643 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0423 19:59:12.557913 3643 cni.go:452] [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key ] I0423 19:59:12.639020 3643 cni.go:473] [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-6594b6fc67-z8lh9 f94837ce702c406e199253a257b45dc888b1c6d95f7cf81d1799ba9679059b84 network default NAD default NAD key default], result "{}", err I0423 19:59:12.677367 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6594b6fc67-z8lh9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:12.677443 3643 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6594b6fc67-z8lh9 I0423 19:59:12.678910 3643 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6594b6fc67-z8lh9, ips: 10.129.0.5 I0423 19:59:12.678943 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6594b6fc67-z8lh9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:12.689343 3643 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7f4c799f6-g6vsz] creating logical port openshift-image-registry_image-registry-7f4c799f6-g6vsz for pod on switch ip-10-0-1-149.ec2.internal I0423 19:59:12.689447 3643 kube.go:256] Updating pod openshift-image-registry/image-registry-7f4c799f6-g6vsz I0423 19:59:12.707542 3643 pod.go:62] [openshift-image-registry/image-registry-7f4c799f6-g6vsz] pod update took 18.128198ms I0423 19:59:12.707569 3643 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7f4c799f6-g6vsz] addLogicalPort annotation time took 18.16218ms I0423 19:59:12.709067 3643 pods.go:271] [openshift-image-registry/image-registry-7f4c799f6-g6vsz] addLogicalPort took 19.741392ms, libovsdb time 1.09538ms I0423 19:59:13.057143 3643 cni.go:452] [openshift-image-registry/image-registry-7f4c799f6-g6vsz 1cb42ec506d754ea854f7e977c68578b69b723d833e466b265ccbca4ed65eb81 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7f4c799f6-g6vsz 1cb42ec506d754ea854f7e977c68578b69b723d833e466b265ccbca4ed65eb81 network default NAD default NAD key ] I0423 19:59:13.067222 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7f4c799f6-g6vsz, hostIfaceName: 1cb42ec506d754e, network: default, NAD default, SandboxID: "1cb42ec506d754ea854f7e977c68578b69b723d833e466b265ccbca4ed65eb81", PCI device ID: , UID: "487092d6-2127-42ca-a84e-d09553446647", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0423 19:59:13.124948 3643 cni.go:473] [openshift-image-registry/image-registry-7f4c799f6-g6vsz 1cb42ec506d754ea854f7e977c68578b69b723d833e466b265ccbca4ed65eb81 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7f4c799f6-g6vsz 1cb42ec506d754ea854f7e977c68578b69b723d833e466b265ccbca4ed65eb81 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:32:55:55:47:2b\",\"name\":\"1cb42ec506d754e\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cff8784-0205-4838-945a-d9ba66e5d5e0\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 19:59:13.151325 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151345 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151350 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151356 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151368 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151373 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151377 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:13.151381 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0423 19:59:13.672622 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6594b6fc67-z8lh9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:13.672650 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6594b6fc67-z8lh9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:13.679505 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6594b6fc67-z8lh9 I0423 19:59:13.679540 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6594b6fc67-z8lh9 W0423 19:59:18.667794 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:18.667816 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:18.667821 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:18.667827 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 19:59:25.737892 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0423 19:59:25.777040 3643 obj_retry.go:498] Detected object openshift-console/console-d74f58697-ptjtc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:25.777563 3643 obj_retry.go:498] Detected object openshift-console/console-d74f58697-ptjtc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:26.219814 3643 obj_retry.go:492] Detected object openshift-console/console-d74f58697-ptjtc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:26.219836 3643 obj_retry.go:492] Detected object openshift-console/console-d74f58697-ptjtc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:26.226673 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-d74f58697-ptjtc I0423 19:59:26.226706 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-d74f58697-ptjtc W0423 19:59:28.140115 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:59:28.140143 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:59:30.762868 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0423 19:59:31.360061 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.360091 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.360099 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.360114 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.360120 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.360126 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.379872 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.379890 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.379896 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 19:59:31.640295 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 19:59:34.668816 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0423 19:59:35.418265 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6594b6fc67-tgd29 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:35.418838 3643 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6594b6fc67-tgd29 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:35.779067 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6594b6fc67-tgd29 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:35.779094 3643 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6594b6fc67-tgd29 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:35.785958 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6594b6fc67-tgd29 I0423 19:59:35.785984 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6594b6fc67-tgd29 I0423 19:59:35.984338 3643 obj_retry.go:498] Detected object openshift-console/console-6bcc7cdd95-xjjnr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:35.984855 3643 obj_retry.go:498] Detected object openshift-console/console-6bcc7cdd95-xjjnr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:36.805180 3643 obj_retry.go:492] Detected object openshift-console/console-6bcc7cdd95-xjjnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:36.805198 3643 obj_retry.go:492] Detected object openshift-console/console-6bcc7cdd95-xjjnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:36.811559 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bcc7cdd95-xjjnr I0423 19:59:36.811587 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bcc7cdd95-xjjnr W0423 19:59:40.816565 3643 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0423 19:59:59.417576 3643 obj_retry.go:498] Detected object openshift-ingress/router-default-5d8c68cc65-h6b7j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:59.418065 3643 obj_retry.go:498] Detected object openshift-ingress/router-default-5d8c68cc65-h6b7j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 19:59:59.888232 3643 obj_retry.go:492] Detected object openshift-ingress/router-default-5d8c68cc65-h6b7j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:59.888257 3643 obj_retry.go:492] Detected object openshift-ingress/router-default-5d8c68cc65-h6b7j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 19:59:59.894662 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5d8c68cc65-h6b7j I0423 19:59:59.894690 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5d8c68cc65-h6b7j I0423 20:00:00.219963 3643 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29616240-fg29g for pod on switch ip-10-0-1-149.ec2.internal I0423 20:00:00.220052 3643 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g I0423 20:00:00.234725 3643 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g] pod update took 14.691322ms I0423 20:00:00.234751 3643 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g] addLogicalPort annotation time took 14.725898ms I0423 20:00:00.235920 3643 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g] addLogicalPort took 15.97579ms, libovsdb time 848.258µs I0423 20:00:00.661323 3643 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key ] I0423 20:00:00.670764 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29616240-fg29g, hostIfaceName: f14044a82c9f014, network: default, NAD default, SandboxID: "f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367", PCI device ID: , UID: "da67ef76-07fa-4bd3-b39c-ff4530b67566", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0423 20:00:00.721198 3643 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:68:8a:41:15:7a\",\"name\":\"f14044a82c9f014\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/02202188-01bd-47f8-ac1e-e3b2d1fe91c4\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 20:00:01.492752 3643 cni.go:452] [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key ] I0423 20:00:01.567491 3643 cni.go:473] [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-d74f58697-25ltm 5312bf4826e338613a02f00048e9e773131ef928d05e4a1cdb8b233e5dab514b network default NAD default NAD key default], result "{}", err I0423 20:00:01.635764 3643 obj_retry.go:498] Detected object openshift-console/console-d74f58697-25ltm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:01.635801 3643 pods.go:176] Deleting pod: openshift-console/console-d74f58697-25ltm I0423 20:00:01.637087 3643 pods.go:236] Attempting to release IPs for pod: openshift-console/console-d74f58697-25ltm, ips: 10.129.0.14 I0423 20:00:01.637117 3643 obj_retry.go:498] Detected object openshift-console/console-d74f58697-25ltm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:01.850999 3643 obj_retry.go:492] Detected object openshift-console/console-d74f58697-25ltm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:01.851021 3643 obj_retry.go:492] Detected object openshift-console/console-d74f58697-25ltm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:01.857595 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-d74f58697-25ltm I0423 20:00:01.857613 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-d74f58697-25ltm I0423 20:00:06.766949 3643 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 for pod on switch ip-10-0-1-149.ec2.internal I0423 20:00:06.767044 3643 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 I0423 20:00:06.780904 3643 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25] pod update took 13.889219ms I0423 20:00:06.780927 3643 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25] addLogicalPort annotation time took 13.916628ms I0423 20:00:06.782142 3643 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25] addLogicalPort took 15.209805ms, libovsdb time 913.874µs I0423 20:00:07.123606 3643 cni.go:452] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key ] I0423 20:00:07.132849 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25, hostIfaceName: c2ee9ecdcb14115, network: default, NAD default, SandboxID: "c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc", PCI device ID: , UID: "33af8796-bf5e-40eb-9c0a-69a9233f6c5a", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0423 20:00:07.182549 3643 cni.go:473] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:47:ed:d0:10:c1\",\"name\":\"c2ee9ecdcb14115\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b4566160-04c2-4e3e-90cc-6878606a7a82\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 20:00:17.910919 3643 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key ] I0423 20:00:17.992354 3643 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g f14044a82c9f0143787d23c6383529f8274173bd826ca04c04bdea424a16b367 network default NAD default NAD key default], result "{}", err I0423 20:00:18.033144 3643 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:18.033182 3643 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g I0423 20:00:18.034429 3643 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g, ips: 10.129.0.20 I0423 20:00:18.034467 3643 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:18.953870 3643 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:18.953899 3643 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616240-fg29g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:21.006749 3643 cni.go:452] [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key ] I0423 20:00:21.100968 3643 cni.go:473] [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 7418cc66df3bd0900495c77bc763671b9826cb43782e8d01373baca2dac1aa68 network default NAD default NAD key default], result "{}", err I0423 20:00:21.162210 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:21.162292 3643 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0423 20:00:21.163927 3643 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.129.0.15 I0423 20:00:21.163971 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:21.926760 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:21.926818 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:21.935043 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0423 20:00:21.935073 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0423 20:00:21.976097 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-149.ec2.internal I0423 20:00:21.976330 3643 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0423 20:00:22.007633 3643 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 31.362929ms I0423 20:00:22.007696 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 31.418498ms I0423 20:00:22.009039 3643 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 32.967278ms, libovsdb time 966.687µs I0423 20:00:22.343992 3643 cni.go:452] [openshift-monitoring/alertmanager-main-1 771be54059d249fdeefc6ba3a4adffecad63c8052f7342885b117b6171880686 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 771be54059d249fdeefc6ba3a4adffecad63c8052f7342885b117b6171880686 network default NAD default NAD key ] I0423 20:00:22.354701 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 771be54059d249f, network: default, NAD default, SandboxID: "771be54059d249fdeefc6ba3a4adffecad63c8052f7342885b117b6171880686", PCI device ID: , UID: "1140f24c-8673-4c02-9858-ab1015e399dc", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0423 20:00:22.405646 3643 cni.go:473] [openshift-monitoring/alertmanager-main-1 771be54059d249fdeefc6ba3a4adffecad63c8052f7342885b117b6171880686 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 771be54059d249fdeefc6ba3a4adffecad63c8052f7342885b117b6171880686 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:ee:1a:31:13:25\",\"name\":\"771be54059d249f\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4527a8d6-8892-4ec7-ad84-1ef9e23d47a4\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 20:00:23.963191 3643 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-dff99c646-vsfqx] creating logical port openshift-monitoring_telemeter-client-dff99c646-vsfqx for pod on switch ip-10-0-1-149.ec2.internal I0423 20:00:23.963333 3643 kube.go:256] Updating pod openshift-monitoring/telemeter-client-dff99c646-vsfqx I0423 20:00:23.987900 3643 pod.go:62] [openshift-monitoring/telemeter-client-dff99c646-vsfqx] pod update took 24.592735ms I0423 20:00:23.987925 3643 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-dff99c646-vsfqx] addLogicalPort annotation time took 24.623762ms I0423 20:00:23.989142 3643 pods.go:271] [openshift-monitoring/telemeter-client-dff99c646-vsfqx] addLogicalPort took 25.967731ms, libovsdb time 907.55µs I0423 20:00:24.324620 3643 cni.go:452] [openshift-monitoring/telemeter-client-dff99c646-vsfqx 13e595e3c46fdab1a35c8ac575f644154a8c14480b638d698dadd79b35424c5f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-dff99c646-vsfqx 13e595e3c46fdab1a35c8ac575f644154a8c14480b638d698dadd79b35424c5f network default NAD default NAD key ] I0423 20:00:24.335102 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-dff99c646-vsfqx, hostIfaceName: 13e595e3c46fdab, network: default, NAD default, SandboxID: "13e595e3c46fdab1a35c8ac575f644154a8c14480b638d698dadd79b35424c5f", PCI device ID: , UID: "8f822285-fb85-4ba5-a853-7ca7b018062d", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0423 20:00:24.390626 3643 cni.go:473] [openshift-monitoring/telemeter-client-dff99c646-vsfqx 13e595e3c46fdab1a35c8ac575f644154a8c14480b638d698dadd79b35424c5f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-dff99c646-vsfqx 13e595e3c46fdab1a35c8ac575f644154a8c14480b638d698dadd79b35424c5f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:73:2b:c3:ee:2d\",\"name\":\"13e595e3c46fdab\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9cd0a47-6896-4fd9-a287-8aa10375b709\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 20:00:25.600522 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:25.601115 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:26.002844 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:26.002872 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:26.011986 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0423 20:00:26.012015 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0423 20:00:29.235482 3643 base_network_controller_pods.go:487] [default/openshift-console/console-84d78d6f76-8vqzr] creating logical port openshift-console_console-84d78d6f76-8vqzr for pod on switch ip-10-0-1-149.ec2.internal I0423 20:00:29.235693 3643 kube.go:256] Updating pod openshift-console/console-84d78d6f76-8vqzr I0423 20:00:29.261318 3643 pod.go:62] [openshift-console/console-84d78d6f76-8vqzr] pod update took 25.666215ms I0423 20:00:29.261347 3643 base_network_controller_pods.go:951] [default/openshift-console/console-84d78d6f76-8vqzr] addLogicalPort annotation time took 25.699559ms I0423 20:00:29.262828 3643 pods.go:271] [openshift-console/console-84d78d6f76-8vqzr] addLogicalPort took 27.367259ms, libovsdb time 1.062052ms I0423 20:00:29.591348 3643 cni.go:452] [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key ] I0423 20:00:29.600732 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-84d78d6f76-8vqzr, hostIfaceName: 00a178b290b43b2, network: default, NAD default, SandboxID: "00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb", PCI device ID: , UID: "9899b3dc-f95a-4dbf-b897-dce61808b446", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0423 20:00:29.652047 3643 cni.go:473] [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:36:e1:09:2c:7e\",\"name\":\"00a178b290b43b2\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/022d4d61-67b1-43e0-9b89-91e247f8043c\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0423 20:00:29.979585 3643 cni.go:452] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key ] I0423 20:00:30.051205 3643 cni.go:473] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 c2ee9ecdcb141155423692a71f384007d154479a857c95b881bae034af6e77fc network default NAD default NAD key default], result "{}", err I0423 20:00:30.090731 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:30.090771 3643 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 I0423 20:00:30.092133 3643 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25, ips: 10.129.0.21 I0423 20:00:30.092166 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:30.092188 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 20:00:30.092202 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0423 20:00:30.092397 3643 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 20:00:30.092413 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0423 20:00:31.119625 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:31.119649 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:31.119666 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:31.119677 3643 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390v6x25 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:31.846461 3643 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 20:00:31.858057 3643 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0423 20:00:40.027658 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:00:42.391929 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:00:42.391953 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:00:42.391959 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 20:00:43.696826 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:43.697424 3643 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:44.536655 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:44.536676 3643 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:44.544329 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0423 20:00:44.544357 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0423 20:00:54.535665 3643 obj_retry.go:498] Detected object openshift-console/console-69b866469c-9fxb6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:54.536247 3643 obj_retry.go:498] Detected object openshift-console/console-69b866469c-9fxb6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:00:55.158485 3643 obj_retry.go:492] Detected object openshift-console/console-69b866469c-9fxb6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:55.158506 3643 obj_retry.go:492] Detected object openshift-console/console-69b866469c-9fxb6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:00:55.166708 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-69b866469c-9fxb6 I0423 20:00:55.166727 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-69b866469c-9fxb6 I0423 20:01:05.362910 3643 obj_retry.go:498] Detected object openshift-console/console-69b866469c-9r4r5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:01:05.363466 3643 obj_retry.go:498] Detected object openshift-console/console-69b866469c-9r4r5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:01:05.622615 3643 obj_retry.go:492] Detected object openshift-console/console-69b866469c-9r4r5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:01:05.622638 3643 obj_retry.go:492] Detected object openshift-console/console-69b866469c-9r4r5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:01:05.627359 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-69b866469c-9r4r5 I0423 20:01:05.627381 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-69b866469c-9r4r5 I0423 20:01:09.131555 3643 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 20:01:28.606798 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:01:28.607362 3643 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:01:28.721294 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:01:28.721314 3643 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:01:28.728850 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0423 20:01:28.728867 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0423 20:03:08.236408 3643 obj_retry.go:498] Detected object openshift-console/console-84d78d6f76-hmhcs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:03:08.237025 3643 obj_retry.go:498] Detected object openshift-console/console-84d78d6f76-hmhcs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:03:08.669675 3643 obj_retry.go:492] Detected object openshift-console/console-84d78d6f76-hmhcs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:03:08.669706 3643 obj_retry.go:492] Detected object openshift-console/console-84d78d6f76-hmhcs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:03:08.674687 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84d78d6f76-hmhcs I0423 20:03:08.674708 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84d78d6f76-hmhcs I0423 20:03:19.263088 3643 cni.go:452] [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key ] I0423 20:03:19.346110 3643 cni.go:473] [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-84d78d6f76-8vqzr 00a178b290b43b24110e3f3f098a082ee7fddf7c075629f53ab65fb29afd0ccb network default NAD default NAD key default], result "{}", err I0423 20:03:19.395517 3643 obj_retry.go:498] Detected object openshift-console/console-84d78d6f76-8vqzr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:03:19.395556 3643 pods.go:176] Deleting pod: openshift-console/console-84d78d6f76-8vqzr I0423 20:03:19.396715 3643 pods.go:236] Attempting to release IPs for pod: openshift-console/console-84d78d6f76-8vqzr, ips: 10.129.0.24 I0423 20:03:19.396740 3643 obj_retry.go:498] Detected object openshift-console/console-84d78d6f76-8vqzr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:03:19.627042 3643 obj_retry.go:492] Detected object openshift-console/console-84d78d6f76-8vqzr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:03:19.627072 3643 obj_retry.go:492] Detected object openshift-console/console-84d78d6f76-8vqzr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:03:19.633846 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84d78d6f76-8vqzr I0423 20:03:19.633876 3643 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84d78d6f76-8vqzr I0423 20:03:21.501376 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0423 20:03:27.622420 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0423 20:04:04.622703 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0423 20:04:28.501619 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0423 20:04:29.599363 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0423 20:04:36.588817 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0423 20:04:46.499074 3643 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=139 I0423 20:04:50.604391 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0423 20:05:33.496162 3643 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-v5pnx for pod on switch ip-10-0-1-149.ec2.internal I0423 20:05:33.496330 3643 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx I0423 20:05:33.527254 3643 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx] pod update took 30.956645ms I0423 20:05:33.527291 3643 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx] addLogicalPort annotation time took 31.021081ms I0423 20:05:33.528560 3643 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx] addLogicalPort took 32.411725ms, libovsdb time 890.555µs I0423 20:05:33.855870 3643 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx 8fc9346f998bad16c865f555675a822b5f0bcc541a3266ab6455d202cdac3609 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx 8fc9346f998bad16c865f555675a822b5f0bcc541a3266ab6455d202cdac3609 network default NAD default NAD key ] I0423 20:05:33.865333 3643 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-v5pnx, hostIfaceName: 8fc9346f998bad1, network: default, NAD default, SandboxID: "8fc9346f998bad16c865f555675a822b5f0bcc541a3266ab6455d202cdac3609", PCI device ID: , UID: "890744bb-f039-4178-aea8-01cb8d807701", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0423 20:05:33.917142 3643 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx 8fc9346f998bad16c865f555675a822b5f0bcc541a3266ab6455d202cdac3609 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-v5pnx 8fc9346f998bad16c865f555675a822b5f0bcc541a3266ab6455d202cdac3609 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:f9:e4:70:28:e0\",\"name\":\"8fc9346f998bad1\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2c2b09c-3466-41c9-a303-d0102b970c57\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0423 20:05:37.167692 3643 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0423 20:05:48.183198 3643 util.go:959] Failed to build node endpoints for node ip-10-0-1-149.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:05:48.215723 3643 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0423 20:05:48.215807 3643 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-bdvf2 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-bdvf2: no pod IPs found I0423 20:05:57.610197 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0423 20:06:12.593947 3643 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0423 20:06:25.773999 3643 namespace.go:98] [tsf] adding namespace I0423 20:06:25.774957 3643 namespace.go:102] [tsf] adding namespace took 927.661µs I0423 20:06:25.797244 3643 namespace.go:142] [tsf] updating namespace I0423 20:06:25.808202 3643 namespace.go:142] [tsf] updating namespace