+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-22.ec2.internal ++ [[ -n ip-10-0-2-22.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-22.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0423 20:43:30.398255842 - disable conntrack on geneve port' I0423 20:43:30.398255842 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0423 20:43:30.415765830 - starting ovnkube-node + echo 'I0423 20:43:30.415765830 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-22.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-22.ec2.internal --init-node ip-10-0-2-22.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-22.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0423 20:43:30.457070 3825 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0423 20:43:30.457125 3825 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-64c91c14e1.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0423 20:43:30.458543 3825 kube.go:431] Waiting for certificate I0423 20:43:30.458581 3825 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0423 20:43:30.458608 3825 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0423 20:43:30.459338 3825 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 20:43:30.494696 3825 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:30.494718 3825 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:30.498643 3825 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:30.498665 3825 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-h88fn" I0423 20:43:30.506504 3825 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-h88fn" I0423 20:43:30.506583 3825 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0423 20:43:31.458917 3825 kube.go:438] Certificate found I0423 20:43:31.459415 3825 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0423 20:43:31.460371 3825 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0423 20:43:31.511952 3825 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-24 20:38:30 +0000 UTC" deadline="2026-04-24 17:14:34.094533542 +0000 UTC" I0423 20:43:31.511990 3825 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h31m2.582555844s" I0423 20:43:31.540655 3825 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 20:43:31.551570 3825 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0423 20:43:31.551618 3825 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0423 20:43:31.551665 3825 server.go:101] MetricServer registers OVS metrics I0423 20:43:31.551739 3825 factory.go:561] Starting watch factory I0423 20:43:31.551869 3825 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.551895 3825 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.552121 3825 server.go:105] MetricServer registers OVN DB metrics I0423 20:43:31.552900 3825 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.552974 3825 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.553132 3825 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.553196 3825 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.553675 3825 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.553692 3825 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.553763 3825 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.553792 3825 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.554325 3825 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.554366 3825 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.556357 3825 controller_manager.go:370] Starting the ovnkube controller I0423 20:43:31.556368 3825 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-22.ec2.internal I0423 20:43:31.556419 3825 controller_manager.go:395] NBDB zone sync took: 44.066µs I0423 20:43:31.556430 3825 factory.go:561] Starting watch factory I0423 20:43:31.557384 3825 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0423 20:43:31.557404 3825 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0423 20:43:31.572812 3825 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0423 20:43:31.572844 3825 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0423 20:43:31.576082 3825 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.576265 3825 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.576690 3825 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.576813 3825 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.576846 3825 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0423 20:43:31.576948 3825 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.576999 3825 server.go:109] MetricServer registers OVN Controller metrics I0423 20:43:31.582508 3825 server.go:113] MetricServer registers OVN Northd metrics I0423 20:43:31.587219 3825 metrics.go:519] OVN Metrics Server starts to run ... I0423 20:43:31.589058 3825 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0423 20:43:31.657886 3825 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 20:43:31.657907 3825 factory.go:1977] *v1.Pod informer cache synced successfully I0423 20:43:31.657911 3825 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 20:43:31.657914 3825 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 20:43:31.657917 3825 factory.go:1977] *v1.Node informer cache synced successfully I0423 20:43:31.657920 3825 factory.go:1977] *v1.Service informer cache synced successfully I0423 20:43:31.658013 3825 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:31.658034 3825 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:31.658044 3825 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:31.658065 3825 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:31.661601 3825 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:31.661610 3825 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0423 20:43:31.662327 3825 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0423 20:43:31.662338 3825 factory.go:1977] *v1.Pod informer cache synced successfully I0423 20:43:31.662341 3825 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0423 20:43:31.662344 3825 factory.go:1977] *v1.Namespace informer cache synced successfully I0423 20:43:31.662347 3825 factory.go:1977] *v1.Node informer cache synced successfully I0423 20:43:31.662350 3825 factory.go:1977] *v1.Service informer cache synced successfully I0423 20:43:31.663477 3825 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 20:43:31.663486 3825 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 20:43:31.663542 3825 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.663556 3825 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.666170 3825 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.668220 3825 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0423 20:43:31.668230 3825 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0423 20:43:31.669324 3825 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 20:43:31.669387 3825 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.669398 3825 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.671850 3825 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.673863 3825 factory.go:1977] *v1.EgressIP informer cache synced successfully I0423 20:43:31.674951 3825 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 20:43:31.675008 3825 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.675022 3825 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.677496 3825 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.679478 3825 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0423 20:43:31.680615 3825 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 20:43:31.680694 3825 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.680705 3825 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.683035 3825 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.685080 3825 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0423 20:43:31.686228 3825 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 20:43:31.686286 3825 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.686314 3825 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.688741 3825 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.690659 3825 factory.go:1977] *v1.EgressService informer cache synced successfully I0423 20:43:31.691798 3825 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 20:43:31.691959 3825 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 20:43:31.691974 3825 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 20:43:31.694484 3825 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0423 20:43:31.696470 3825 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0423 20:43:31.697760 3825 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 20:43:31.697824 3825 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.697838 3825 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.697875 3825 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.697894 3825 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.700451 3825 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.700598 3825 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0423 20:43:31.701530 3825 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0423 20:43:31.703247 3825 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 20:43:31.703264 3825 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 20:43:31.703273 3825 factory.go:680] Watch Factory start up complete, took: 151.538492ms I0423 20:43:31.703337 3825 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0423 20:43:31.707063 3825 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0423 20:43:31.707075 3825 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0423 20:43:31.707083 3825 factory.go:680] Watch Factory start up complete, took: 150.652019ms I0423 20:43:31.707090 3825 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-22.ec2.internal" zone I0423 20:43:31.757669 3825 default_node_network_controller.go:710] Initializing the default node network controller I0423 20:43:31.832250 3825 udn_isolation.go:101] Starting UDN host isolation manager I0423 20:43:31.838435 3825 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0423 20:43:31.905756 3825 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0423 20:43:31.905890 3825 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0423 20:43:31.905908 3825 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0423 20:43:31.938784 3825 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0423 20:43:31.938856 3825 default_node_network_controller.go:809] Node ip-10-0-2-22.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0423 20:43:31.944595 3825 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.22"] k8s.ovn.org/zone-name:ip-10-0-2-22.ec2.internal] on node ip-10-0-2-22.ec2.internal I0423 20:43:31.966665 3825 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0423 20:43:31.974137 3825 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0423 20:43:31.974302 3825 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0423 20:43:31.974476 3825 gateway_init.go:263] Preparing Gateway I0423 20:43:31.974491 3825 gateway_shared_intf.go:1687] Creating new gateway I0423 20:43:32.084252 3825 iptables.go:108] Creating table: filter chain: FORWARD I0423 20:43:32.091166 3825 iptables.go:108] Creating table: filter chain: OUTPUT I0423 20:43:32.098014 3825 gateway_shared_intf.go:1816] Gateway Creation Complete I0423 20:43:32.098029 3825 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-22.ec2.internal","mac-address":"0e:97:79:e6:5b:db","ip-addresses":["10.0.2.22/24"],"ip-address":"10.0.2.22/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:182fb25b-9e98-4261-a3e5-06057c519e6f k8s.ovn.org/node-encap-ips:["10.0.2.22"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.22/24"} k8s.ovn.org/zone-name:ip-10-0-2-22.ec2.internal] on node ip-10-0-2-22.ec2.internal I0423 20:43:32.117855 3825 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:32.207294 3825 controller_manager.go:429] Waiting for node in zone sync took: 500.189779ms I0423 20:43:32.212488 3825 controller_manager.go:611] Switching to transit router for layer2 networks I0423 20:43:32.212506 3825 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-22.ec2.internal I0423 20:43:32.228416 3825 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0423 20:43:32.235264 3825 controller_manager.go:290] SCTP support detected in OVN I0423 20:43:32.236057 3825 services_controller.go:69] Creating services controller for network=default I0423 20:43:32.236145 3825 udn_enabled_svc.go:80] Waiting for service informer to sync I0423 20:43:32.236675 3825 udn_enabled_svc.go:95] Performing full resync I0423 20:43:32.236751 3825 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0423 20:43:32.236856 3825 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0423 20:43:32.236875 3825 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0423 20:43:32.236887 3825 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 20:43:32.237061 3825 udn_enabled_svc.go:99] Waiting for handler to sync I0423 20:43:32.237063 3825 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0423 20:43:32.237072 3825 udn_enabled_svc.go:104] Starting worker I0423 20:43:32.237111 3825 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0423 20:43:32.237145 3825 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 57.24µs I0423 20:43:32.237152 3825 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 32.034µs I0423 20:43:32.237159 3825 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 47.951µs I0423 20:43:32.237164 3825 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0423 20:43:32.237174 3825 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0423 20:43:32.237183 3825 default_network_controller.go:325] Starting the default network controller I0423 20:43:32.237197 3825 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 11.961µs I0423 20:43:32.246888 3825 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0423 20:43:32.246947 3825 repair.go:33] Syncing exgw routes took 47.442µs I0423 20:43:32.246969 3825 default_network_controller.go:414] Starting all the Watchers... I0423 20:43:32.247151 3825 namespace.go:98] [openshift-operators] adding namespace I0423 20:43:32.247204 3825 namespace.go:98] [openshift-backplane-tam] adding namespace I0423 20:43:32.247210 3825 namespace.go:98] [openshift-route-controller-manager] adding namespace I0423 20:43:32.247222 3825 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0423 20:43:32.247229 3825 namespace.go:98] [openshift-infra] adding namespace I0423 20:43:32.247303 3825 namespace.go:98] [openshift-network-diagnostics] adding namespace I0423 20:43:32.247312 3825 namespace.go:98] [dedicated-admin] adding namespace I0423 20:43:32.247316 3825 namespace.go:98] [openshift-etcd] adding namespace I0423 20:43:32.247329 3825 namespace.go:98] [openshift-dns-operator] adding namespace I0423 20:43:32.247334 3825 namespace.go:98] [openshift-backplane-cee] adding namespace I0423 20:43:32.247340 3825 namespace.go:98] [openshift] adding namespace I0423 20:43:32.247345 3825 namespace.go:98] [kube-public] adding namespace I0423 20:43:32.247351 3825 namespace.go:98] [openshift-image-registry] adding namespace I0423 20:43:32.247355 3825 namespace.go:98] [openshift-machine-config-operator] adding namespace I0423 20:43:32.247364 3825 namespace.go:98] [openshift-logging] adding namespace I0423 20:43:32.247739 3825 namespace.go:102] [openshift-operators] adding namespace took 576.501µs I0423 20:43:32.247754 3825 namespace.go:98] [openshift-multus] adding namespace W0423 20:43:32.247815 3825 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-khgmn: pod openshift-multus/network-metrics-daemon-khgmn: no pod IPs found I0423 20:43:32.248341 3825 namespace.go:102] [openshift-multus] adding namespace took 581.247µs I0423 20:43:32.248360 3825 namespace.go:98] [openshift-controller-manager] adding namespace I0423 20:43:32.248880 3825 namespace.go:102] [openshift-controller-manager] adding namespace took 513.194µs I0423 20:43:32.248896 3825 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0423 20:43:32.249335 3825 namespace.go:102] [openshift-backplane-tam] adding namespace took 2.120271ms I0423 20:43:32.249355 3825 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0423 20:43:32.249818 3825 namespace.go:102] [openshift-route-controller-manager] adding namespace took 2.601729ms I0423 20:43:32.249831 3825 namespace.go:98] [openshift-service-ca-operator] adding namespace I0423 20:43:32.250306 3825 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 3.07195ms I0423 20:43:32.250326 3825 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0423 20:43:32.250803 3825 namespace.go:102] [openshift-infra] adding namespace took 3.568356ms W0423 20:43:32.250805 3825 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rw6vl: pod openshift-network-diagnostics/network-check-target-rw6vl: no pod IPs found I0423 20:43:32.250816 3825 namespace.go:98] [openshift-apiserver] adding namespace I0423 20:43:32.251327 3825 namespace.go:102] [openshift-network-diagnostics] adding namespace took 4.015528ms I0423 20:43:32.251342 3825 namespace.go:98] [openshift-console-user-settings] adding namespace I0423 20:43:32.251780 3825 namespace.go:102] [dedicated-admin] adding namespace took 4.462007ms I0423 20:43:32.251800 3825 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0423 20:43:32.252249 3825 namespace.go:102] [openshift-etcd] adding namespace took 4.927294ms I0423 20:43:32.252262 3825 namespace.go:98] [openshift-config-managed] adding namespace I0423 20:43:32.253145 3825 namespace.go:102] [openshift-dns-operator] adding namespace took 5.809126ms I0423 20:43:32.253162 3825 namespace.go:98] [openshift-insights] adding namespace I0423 20:43:32.253597 3825 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.257003ms I0423 20:43:32.253614 3825 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0423 20:43:32.254061 3825 namespace.go:102] [openshift] adding namespace took 6.715647ms I0423 20:43:32.254073 3825 namespace.go:98] [openshift-apiserver-operator] adding namespace I0423 20:43:32.254483 3825 namespace.go:102] [kube-public] adding namespace took 7.132013ms I0423 20:43:32.254500 3825 namespace.go:98] [openshift-must-gather-operator] adding namespace I0423 20:43:32.255021 3825 namespace.go:102] [openshift-image-registry] adding namespace took 7.663013ms I0423 20:43:32.255035 3825 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0423 20:43:32.255451 3825 namespace.go:102] [openshift-machine-config-operator] adding namespace took 8.091087ms I0423 20:43:32.255469 3825 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0423 20:43:32.255882 3825 namespace.go:102] [openshift-logging] adding namespace took 8.512832ms I0423 20:43:32.255895 3825 namespace.go:98] [openshift-network-console] adding namespace I0423 20:43:32.256266 3825 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.364272ms I0423 20:43:32.256281 3825 namespace.go:98] [openshift-backplane-cse] adding namespace I0423 20:43:32.256666 3825 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.302307ms I0423 20:43:32.256685 3825 namespace.go:98] [openshift-authentication] adding namespace I0423 20:43:32.257099 3825 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.262945ms I0423 20:43:32.257111 3825 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0423 20:43:32.257498 3825 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.164715ms I0423 20:43:32.257514 3825 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0423 20:43:32.257912 3825 namespace.go:102] [openshift-apiserver] adding namespace took 7.092086ms I0423 20:43:32.257926 3825 namespace.go:98] [openshift-dns] adding namespace I0423 20:43:32.258288 3825 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.93948ms I0423 20:43:32.258301 3825 namespace.go:98] [openshift-node] adding namespace I0423 20:43:32.258648 3825 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.84162ms I0423 20:43:32.258660 3825 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0423 20:43:32.259096 3825 namespace.go:102] [openshift-config-managed] adding namespace took 6.828343ms I0423 20:43:32.259110 3825 namespace.go:98] [openshift-cluster-version] adding namespace I0423 20:43:32.259459 3825 namespace.go:102] [openshift-insights] adding namespace took 6.291557ms I0423 20:43:32.259473 3825 namespace.go:98] [openshift-host-network] adding namespace I0423 20:43:32.259873 3825 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.253841ms I0423 20:43:32.259888 3825 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0423 20:43:32.260240 3825 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.162383ms I0423 20:43:32.260253 3825 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0423 20:43:32.260613 3825 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.10703ms I0423 20:43:32.260644 3825 namespace.go:98] [openshift-ingress-canary] adding namespace I0423 20:43:32.261027 3825 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 5.987625ms I0423 20:43:32.261040 3825 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0423 20:43:32.261418 3825 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 5.943685ms I0423 20:43:32.261428 3825 namespace.go:98] [openshift-console-operator] adding namespace I0423 20:43:32.261972 3825 namespace.go:102] [openshift-network-console] adding namespace took 6.071317ms I0423 20:43:32.261985 3825 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0423 20:43:32.262352 3825 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.064988ms I0423 20:43:32.262367 3825 namespace.go:98] [openshift-marketplace] adding namespace I0423 20:43:32.262756 3825 namespace.go:102] [openshift-authentication] adding namespace took 6.063641ms I0423 20:43:32.262771 3825 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0423 20:43:32.263173 3825 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.055264ms I0423 20:43:32.263190 3825 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0423 20:43:32.263615 3825 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.095926ms I0423 20:43:32.263641 3825 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0423 20:43:32.264017 3825 namespace.go:102] [openshift-dns] adding namespace took 6.084214ms I0423 20:43:32.264034 3825 namespace.go:98] [kube-system] adding namespace I0423 20:43:32.264454 3825 namespace.go:102] [openshift-node] adding namespace took 6.146728ms I0423 20:43:32.264472 3825 namespace.go:98] [openshift-network-node-identity] adding namespace I0423 20:43:32.264862 3825 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.19645ms I0423 20:43:32.264877 3825 namespace.go:98] [openshift-kube-apiserver] adding namespace I0423 20:43:32.265317 3825 namespace.go:102] [openshift-cluster-version] adding namespace took 6.199098ms I0423 20:43:32.265336 3825 namespace.go:98] [openshift-config] adding namespace I0423 20:43:32.265847 3825 namespace.go:102] [openshift-host-network] adding namespace took 6.368589ms I0423 20:43:32.265862 3825 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0423 20:43:32.266266 3825 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.372019ms I0423 20:43:32.266280 3825 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0423 20:43:32.266691 3825 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.431283ms I0423 20:43:32.266704 3825 namespace.go:98] [openshift-backplane-mobb] adding namespace I0423 20:43:32.267125 3825 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.474386ms I0423 20:43:32.267142 3825 namespace.go:98] [openshift-monitoring] adding namespace I0423 20:43:32.267519 3825 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.472468ms I0423 20:43:32.267535 3825 namespace.go:98] [openshift-backplane-srep] adding namespace I0423 20:43:32.267937 3825 namespace.go:102] [openshift-console-operator] adding namespace took 6.503567ms I0423 20:43:32.267953 3825 namespace.go:98] [open-cluster-management-2ps35p4e7fo08ot66mbftq9uvphot0ei] adding namespace I0423 20:43:32.268337 3825 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.345862ms I0423 20:43:32.268352 3825 namespace.go:98] [openshift-ingress] adding namespace I0423 20:43:32.268747 3825 namespace.go:102] [openshift-marketplace] adding namespace took 6.372806ms I0423 20:43:32.269156 3825 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.377569ms I0423 20:43:32.269174 3825 namespace.go:98] [openshift-backplane-csm] adding namespace I0423 20:43:32.269609 3825 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 6.413871ms I0423 20:43:32.269640 3825 namespace.go:98] [openshift-authentication-operator] adding namespace I0423 20:43:32.270049 3825 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.402253ms I0423 20:43:32.270066 3825 namespace.go:98] [openshift-package-operator] adding namespace I0423 20:43:32.270436 3825 namespace.go:102] [kube-system] adding namespace took 6.395274ms I0423 20:43:32.270450 3825 namespace.go:98] [openshift-ingress-operator] adding namespace I0423 20:43:32.270940 3825 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.461741ms I0423 20:43:32.270957 3825 namespace.go:98] [default] adding namespace I0423 20:43:32.271326 3825 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.442807ms I0423 20:43:32.271342 3825 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0423 20:43:32.271737 3825 namespace.go:102] [openshift-config] adding namespace took 6.394167ms I0423 20:43:32.271753 3825 namespace.go:98] [openshift-service-ca] adding namespace I0423 20:43:32.272199 3825 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.3276ms I0423 20:43:32.272217 3825 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0423 20:43:32.272589 3825 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.30097ms I0423 20:43:32.272604 3825 namespace.go:98] [openshift-console] adding namespace I0423 20:43:32.273028 3825 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.318267ms I0423 20:43:32.273046 3825 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0423 20:43:32.273496 3825 namespace.go:102] [openshift-monitoring] adding namespace took 6.34845ms I0423 20:43:32.273507 3825 namespace.go:98] [openshift-config-operator] adding namespace I0423 20:43:32.274298 3825 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.756119ms I0423 20:43:32.274317 3825 namespace.go:98] [openshift-machine-api] adding namespace I0423 20:43:32.274717 3825 namespace.go:102] [open-cluster-management-2ps35p4e7fo08ot66mbftq9uvphot0ei] adding namespace took 6.757086ms I0423 20:43:32.274733 3825 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0423 20:43:32.275231 3825 namespace.go:102] [openshift-ingress] adding namespace took 6.872349ms I0423 20:43:32.275247 3825 namespace.go:98] [openshift-kube-scheduler] adding namespace I0423 20:43:32.275657 3825 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.475174ms I0423 20:43:32.275673 3825 namespace.go:98] [openshift-customer-monitoring] adding namespace I0423 20:43:32.276086 3825 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.438353ms I0423 20:43:32.276492 3825 namespace.go:102] [openshift-package-operator] adding namespace took 6.419665ms I0423 20:43:32.276892 3825 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.435389ms I0423 20:43:32.276908 3825 namespace.go:98] [openshift-network-operator] adding namespace I0423 20:43:32.277274 3825 namespace.go:102] [default] adding namespace took 6.310115ms I0423 20:43:32.277291 3825 namespace.go:98] [kube-node-lease] adding namespace I0423 20:43:32.277696 3825 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.346972ms I0423 20:43:32.277713 3825 namespace.go:98] [openshift-backplane] adding namespace I0423 20:43:32.278131 3825 namespace.go:102] [openshift-service-ca] adding namespace took 6.372323ms I0423 20:43:32.278543 3825 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.318715ms I0423 20:43:32.278953 3825 namespace.go:102] [openshift-console] adding namespace took 6.342376ms I0423 20:43:32.279355 3825 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.303307ms I0423 20:43:32.279371 3825 namespace.go:98] [openshift-operators-redhat] adding namespace I0423 20:43:32.279756 3825 namespace.go:102] [openshift-config-operator] adding namespace took 6.24288ms I0423 20:43:32.280154 3825 namespace.go:102] [openshift-machine-api] adding namespace took 5.830112ms I0423 20:43:32.280566 3825 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 5.824244ms I0423 20:43:32.280963 3825 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.709429ms I0423 20:43:32.281344 3825 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.665796ms I0423 20:43:32.281755 3825 namespace.go:102] [openshift-network-operator] adding namespace took 4.839885ms I0423 20:43:32.282150 3825 namespace.go:102] [kube-node-lease] adding namespace took 4.852916ms I0423 20:43:32.282554 3825 namespace.go:102] [openshift-backplane] adding namespace took 4.835225ms I0423 20:43:32.282913 3825 namespace.go:102] [openshift-operators-redhat] adding namespace took 3.536785ms I0423 20:43:32.283126 3825 master.go:548] Adding or Updating local node "ip-10-0-2-22.ec2.internal" for network "default" I0423 20:43:32.284025 3825 zone_ic_handler.go:195] Time taken to create transit switch: 356.081µs I0423 20:43:32.284038 3825 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-197.ec2.internal for the network default E0423 20:43:32.284243 3825 obj_retry.go:575] Failed to create *v1.Node ip-10-0-0-172.ec2.internal, error: adding or updating remote node IC resources ip-10-0-0-172.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-0-172.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc009eb0478 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u0845505159}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0423 20:43:32.284302 3825 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-172.ec2.internal", UID:"558dde10-52cc-4f64-9a1b-6d702c1e2ac9", APIVersion:"v1", ResourceVersion:"8653", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' adding or updating remote node IC resources ip-10-0-0-172.ec2.internal failed, err - ensuring transit switch for remote zone node ip-10-0-0-172.ec2.internal for the network default failed : err - failed to create/update transit switch transit_switch: error in transact with ops [{Op:wait Table:Logical_Switch Row:map[] Rows:[map[name:transit_switch]] Columns:[name] Mutations:[] Timeout:0xc009eb0478 Where:[where column name == transit_switch] Until:!= Durable: Comment: Lock: UUID: UUIDName:} {Op:insert Table:Logical_Switch Row:map[name:transit_switch other_config:{GoMap:map[interconn-ts:transit_switch mcast_flood_unregistered:true mcast_querier:false mcast_snoop:true requested-tnl-key:16711683]}] Rows:[] Columns:[] Mutations:[] Timeout: Where:[] Until: Durable: Comment: Lock: UUID: UUIDName:u0845505159}] results [{Count:0 Error:timed out Details:"where" clause test failed UUID:{GoUUID:} Rows:[]} {Count:0 Error: Details: UUID:{GoUUID:} Rows:[]}] and errors [timed out: "where" clause test failed]: 1 ovsdb operations failed I0423 20:43:32.285433 3825 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-197.ec2.internal" on network "default" took: 1.775268ms W0423 20:43:32.291160 3825 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-khgmn: pod openshift-multus/network-metrics-daemon-khgmn: no pod IPs found W0423 20:43:32.291179 3825 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-rw6vl: pod openshift-network-diagnostics/network-check-target-rw6vl: no pod IPs found I0423 20:43:32.292174 3825 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-22.ec2.internal for the network default I0423 20:43:32.292366 3825 zone_ic_handler.go:195] Time taken to create transit switch: 180.45µs I0423 20:43:32.293079 3825 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0423 20:43:32.293133 3825 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0423 20:43:32.293148 3825 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0423 20:43:32.293183 3825 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-172.ec2.internal in network "default" I0423 20:43:32.293234 3825 node_tracker.go:169] Node ip-10-0-0-172.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:32.293250 3825 services_controller.go:574] Full service sync requested for network=default I0423 20:43:32.293257 3825 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-197.ec2.internal in network "default" I0423 20:43:32.293279 3825 node_tracker.go:169] Node ip-10-0-1-197.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:32.293286 3825 services_controller.go:574] Full service sync requested for network=default I0423 20:43:32.293290 3825 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0423 20:43:32.293320 3825 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:32.293325 3825 services_controller.go:574] Full service sync requested for network=default I0423 20:43:32.393381 3825 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0423 20:43:32.393405 3825 services_controller.go:208] Setting up event handlers for services for network=default I0423 20:43:32.393544 3825 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0423 20:43:32.393641 3825 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0423 20:43:32.393658 3825 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0423 20:43:32.493920 3825 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0423 20:43:32.493941 3825 repair.go:60] Starting repairing loop for services I0423 20:43:32.494025 3825 repair.go:131] Deleted 0 stale service LBs I0423 20:43:32.494044 3825 repair.go:137] Deleted 0 stale Chassis Template Vars I0423 20:43:32.494061 3825 repair.go:62] Finished repairing loop for services: 122.309µs I0423 20:43:32.494084 3825 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0423 20:43:32.494094 3825 services_controller.go:254] Starting workers for network=default I0423 20:43:32.494681 3825 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-khgmn] creating logical port openshift-multus_network-metrics-daemon-khgmn for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:32.494729 3825 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-rw6vl] creating logical port openshift-network-diagnostics_network-check-target-rw6vl for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:32.494815 3825 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-khgmn I0423 20:43:32.494846 3825 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-rw6vl W0423 20:43:32.498397 3825 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:43:32.498931 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0423 20:43:32.499153 3825 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:32.508805 3825 pod.go:62] [openshift-network-diagnostics/network-check-target-rw6vl] pod update took 13.999596ms I0423 20:43:32.508821 3825 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-rw6vl] addLogicalPort annotation time took 14.019119ms I0423 20:43:32.509857 3825 pods.go:271] [openshift-network-diagnostics/network-check-target-rw6vl] addLogicalPort took 15.13466ms, libovsdb time 718.937µs I0423 20:43:32.514653 3825 pod.go:62] [openshift-multus/network-metrics-daemon-khgmn] pod update took 19.890934ms I0423 20:43:32.514672 3825 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-khgmn] addLogicalPort annotation time took 19.913409ms I0423 20:43:32.515606 3825 pods.go:271] [openshift-multus/network-metrics-daemon-khgmn] addLogicalPort took 20.932663ms, libovsdb time 691.785µs I0423 20:43:32.516118 3825 admin_network_policy_controller.go:227] Starting controller default-network-controller I0423 20:43:32.516142 3825 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0423 20:43:32.516163 3825 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0423 20:43:32.516171 3825 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0423 20:43:32.516349 3825 repair.go:29] Repairing admin network policies took 172.713µs I0423 20:43:32.516459 3825 repair.go:92] Repairing baseline admin network policies took 100.832µs I0423 20:43:32.516822 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0423 20:43:32.516837 3825 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0423 20:43:32.518141 3825 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0423 20:43:32.518161 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0423 20:43:32.518169 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0423 20:43:32.519423 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0423 20:43:32.519447 3825 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0423 20:43:32.519455 3825 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0423 20:43:32.520719 3825 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0423 20:43:32.520762 3825 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0423 20:43:32.520783 3825 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0423 20:43:32.521865 3825 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0423 20:43:32.521887 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0423 20:43:32.521898 3825 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0423 20:43:32.522406 3825 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0423 20:43:32.522423 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0423 20:43:32.522431 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0423 20:43:32.523871 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0423 20:43:32.523890 3825 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0423 20:43:32.523900 3825 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0423 20:43:32.525842 3825 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0423 20:43:32.525860 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0423 20:43:32.525868 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0423 20:43:32.527560 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0423 20:43:32.527580 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0423 20:43:32.528461 3825 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0423 20:43:32.528481 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 20:43:32.529230 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0423 20:43:32.529249 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0423 20:43:32.529257 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0423 20:43:32.529789 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0423 20:43:32.529807 3825 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0423 20:43:32.529818 3825 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0423 20:43:32.531053 3825 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0423 20:43:32.531070 3825 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0423 20:43:32.531076 3825 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0423 20:43:32.531426 3825 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0423 20:43:32.531446 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0423 20:43:32.531455 3825 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0423 20:43:32.531967 3825 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0423 20:43:32.531982 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0423 20:43:32.531988 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0423 20:43:32.532250 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0423 20:43:32.532267 3825 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0423 20:43:32.532278 3825 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0423 20:43:32.533977 3825 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0423 20:43:32.533994 3825 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0423 20:43:32.534002 3825 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0423 20:43:32.534557 3825 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0423 20:43:32.534590 3825 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0423 20:43:32.534603 3825 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0423 20:43:32.535690 3825 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0423 20:43:32.535709 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0423 20:43:32.535728 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0423 20:43:32.536267 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0423 20:43:32.536285 3825 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0423 20:43:32.536292 3825 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0423 20:43:32.537437 3825 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0423 20:43:32.537455 3825 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0423 20:43:32.537464 3825 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0423 20:43:32.538375 3825 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0423 20:43:32.538402 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0423 20:43:32.538412 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0423 20:43:32.538936 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0423 20:43:32.538954 3825 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0423 20:43:32.538961 3825 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0423 20:43:32.539902 3825 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0423 20:43:32.539930 3825 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0423 20:43:32.539942 3825 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0423 20:43:32.541204 3825 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0423 20:43:32.541221 3825 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0423 20:43:32.541229 3825 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0423 20:43:32.542935 3825 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0423 20:43:32.542957 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0423 20:43:32.542966 3825 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0423 20:43:32.543645 3825 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0423 20:43:32.543663 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0423 20:43:32.543680 3825 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0423 20:43:32.544144 3825 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0423 20:43:32.544164 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0423 20:43:32.544178 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0423 20:43:32.544647 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0423 20:43:32.544666 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0423 20:43:32.544674 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0423 20:43:32.545273 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0423 20:43:32.545290 3825 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0423 20:43:32.545312 3825 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0423 20:43:32.546052 3825 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0423 20:43:32.551876 3825 egressfirewall.go:374] Starting EgressFirewall controller I0423 20:43:32.551901 3825 controller.go:133] Adding controller egress-firewall-controller event handlers I0423 20:43:32.551931 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0423 20:43:32.551943 3825 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0423 20:43:32.551951 3825 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0423 20:43:32.552015 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0423 20:43:32.552027 3825 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0423 20:43:32.552296 3825 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0423 20:43:32.552312 3825 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0423 20:43:32.552319 3825 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0423 20:43:32.552327 3825 egressqos.go:193] Setting up event handlers for EgressQoS I0423 20:43:32.552423 3825 egressfirewall.go:723] Syncing node "ip-10-0-0-172.ec2.internal" for egress firewall I0423 20:43:32.552438 3825 egressfirewall.go:723] Syncing node "ip-10-0-1-197.ec2.internal" for egress firewall I0423 20:43:32.552443 3825 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0423 20:43:32.552597 3825 egressqos.go:245] Starting EgressQoS Controller I0423 20:43:32.552612 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0423 20:43:32.552642 3825 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0423 20:43:32.552652 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0423 20:43:32.552659 3825 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0423 20:43:32.552664 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0423 20:43:32.552669 3825 shared_informer.go:356] "Caches are synced" controller="egressqos" I0423 20:43:32.552673 3825 egressqos.go:259] Repairing EgressQoSes I0423 20:43:32.552677 3825 egressqos.go:399] Starting repairing loop for egressqos I0423 20:43:32.552761 3825 egressqos.go:401] Finished repairing loop for egressqos: 83.356µs I0423 20:43:32.552778 3825 egressservice_zone.go:132] Setting up event handlers for Egress Services I0423 20:43:32.552834 3825 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-22.ec2.internal I0423 20:43:32.552911 3825 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-22.ec2.internal : 80.096µs I0423 20:43:32.552960 3825 egressservice_zone.go:210] Starting Egress Services Controller I0423 20:43:32.552994 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 20:43:32.553003 3825 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 20:43:32.553012 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 20:43:32.553020 3825 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 20:43:32.553027 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 20:43:32.553039 3825 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 20:43:32.553050 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0423 20:43:32.553057 3825 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0423 20:43:32.553061 3825 egressservice_zone.go:228] Repairing Egress Services I0423 20:43:32.553299 3825 master_controller.go:88] Starting Admin Policy Based Route Controller I0423 20:43:32.553308 3825 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 20:43:32.553344 3825 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-172.ec2.internal I0423 20:43:32.553361 3825 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-172.ec2.internal: 38.483µs I0423 20:43:32.553368 3825 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-197.ec2.internal I0423 20:43:32.553374 3825 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-197.ec2.internal: 5.562µs I0423 20:43:32.553378 3825 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:32.553396 3825 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 3.795µs I0423 20:43:32.553538 3825 default_network_controller.go:585] Completing all the Watchers took 306.559908ms I0423 20:43:32.553552 3825 default_network_controller.go:589] Starting unidling controllers I0423 20:43:32.553563 3825 unidle.go:46] Registering OVN SB ControllerEvent handler I0423 20:43:32.553569 3825 unidle.go:63] Populating Initial ContollerEvent events I0423 20:43:32.553649 3825 unidle.go:79] Setting up event handlers for services I0423 20:43:32.553825 3825 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0423 20:43:32.619285 3825 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:33.119172 3825 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:33.618905 3825 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:34.119028 3825 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:34.424751 3825 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0423 20:43:34.618259 3825 default_node_network_controller.go:519] Node connection status = connected I0423 20:43:34.636413 3825 bridgeconfig.go:537] Gateway is ready I0423 20:43:34.636433 3825 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0423 20:43:34.648888 3825 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.22. Updating OVN encap IP. I0423 20:43:34.654756 3825 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.22 - it is already configured I0423 20:43:34.654772 3825 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.22/24:{} 172.20.0.1/32:{}]. Updating annotations. I0423 20:43:34.654978 3825 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.22/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-22.ec2.internal","mac-address":"0e:97:79:e6:5b:db","ip-addresses":["10.0.2.22/24"],"ip-address":"10.0.2.22/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:182fb25b-9e98-4261-a3e5-06057c519e6f k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.22/24"}] on node ip-10-0-2-22.ec2.internal I0423 20:43:34.673330 3825 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:34.673350 3825 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 22.624µs I0423 20:43:34.673370 3825 egressfirewall.go:723] Syncing node "ip-10-0-2-22.ec2.internal" for egress firewall I0423 20:43:34.673409 3825 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-22.ec2.internal in network "default" I0423 20:43:34.673457 3825 master.go:548] Adding or Updating local node "ip-10-0-2-22.ec2.internal" for network "default" I0423 20:43:34.673489 3825 node_tracker.go:169] Node ip-10-0-2-22.ec2.internal switch + router changed, syncing services in network "default" I0423 20:43:34.673512 3825 services_controller.go:574] Full service sync requested for network=default W0423 20:43:34.673764 3825 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:43:34.673784 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 20:43:34.674222 3825 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.22 I0423 20:43:34.674373 3825 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-22.ec2.internal I0423 20:43:34.678645 3825 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-22.ec2.internal. Updating no re-route policies I0423 20:43:34.678731 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-rktbx I0423 20:43:34.678757 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-rktbx I0423 20:43:34.678753 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-22.ec2.internal I0423 20:43:34.678767 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-khgmn I0423 20:43:34.678772 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-twmdg I0423 20:43:34.678778 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-22.ec2.internal I0423 20:43:34.678786 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-khgmn I0423 20:43:34.678788 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-twmdg I0423 20:43:34.678790 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-rktbx after 0 failed attempt(s) I0423 20:43:34.678801 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-22.ec2.internal after 0 failed attempt(s) I0423 20:43:34.678814 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-twmdg after 0 failed attempt(s) I0423 20:43:34.678830 3825 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-22.ec2.internal I0423 20:43:34.678833 3825 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-khgmn] creating logical port openshift-multus_network-metrics-daemon-khgmn for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:34.678846 3825 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-22.ec2.internal I0423 20:43:34.678846 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-lxjdh I0423 20:43:34.678868 3825 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-22.ec2.internal after 0 failed attempt(s) I0423 20:43:34.678874 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-lxjdh I0423 20:43:34.678866 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-45fgv I0423 20:43:34.678883 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-z2g7n I0423 20:43:34.678893 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-bxmzq I0423 20:43:34.678892 3825 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-fj6kh I0423 20:43:34.678898 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-lxjdh after 0 failed attempt(s) I0423 20:43:34.678900 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-q2pjw I0423 20:43:34.678900 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-z2g7n I0423 20:43:34.678908 3825 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-fj6kh I0423 20:43:34.678908 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-jzbqs I0423 20:43:34.678914 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-q2pjw I0423 20:43:34.678922 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-z2g7n after 0 failed attempt(s) I0423 20:43:34.678926 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-jzbqs I0423 20:43:34.678934 3825 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-fj6kh after 0 failed attempt(s) I0423 20:43:34.678935 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-q2pjw after 0 failed attempt(s) I0423 20:43:34.678887 3825 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl I0423 20:43:34.678948 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-jzbqs after 0 failed attempt(s) I0423 20:43:34.678953 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl I0423 20:43:34.678975 3825 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-rw6vl] creating logical port openshift-network-diagnostics_network-check-target-rw6vl for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:34.678895 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-45fgv I0423 20:43:34.679015 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-45fgv after 0 failed attempt(s) I0423 20:43:34.678902 3825 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-bxmzq I0423 20:43:34.679048 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-bxmzq after 0 failed attempt(s) I0423 20:43:34.679722 3825 pods.go:271] [openshift-multus/network-metrics-daemon-khgmn] addLogicalPort took 902.292µs, libovsdb time 261.135µs I0423 20:43:34.679735 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-khgmn after 0 failed attempt(s) I0423 20:43:34.679777 3825 pods.go:271] [openshift-network-diagnostics/network-check-target-rw6vl] addLogicalPort took 809.634µs, libovsdb time 300.936µs I0423 20:43:34.679786 3825 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-rw6vl after 0 failed attempt(s) I0423 20:43:34.689283 3825 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0423 20:43:34.702421 3825 iptables.go:108] Creating table: mangle chain: OUTPUT I0423 20:43:34.706215 3825 iptables.go:108] Creating table: nat chain: OUTPUT I0423 20:43:34.765725 3825 iptables.go:108] Creating table: nat chain: PREROUTING I0423 20:43:34.829826 3825 iptables.go:108] Creating table: filter chain: FORWARD I0423 20:43:34.848674 3825 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0423 20:43:34.848782 3825 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0423 20:43:34.848831 3825 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0423 20:43:34.848913 3825 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0423 20:43:34.848966 3825 gateway_init.go:325] Gateway and management port readiness took 2.735425722s I0423 20:43:34.848999 3825 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0423 20:43:34.849047 3825 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0423 20:43:34.849059 3825 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0423 20:43:34.849068 3825 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0423 20:43:34.849134 3825 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0423 20:43:34.849230 3825 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 61.978µs I0423 20:43:34.849536 3825 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0423 20:43:34.849565 3825 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 18.671µs I0423 20:43:34.849573 3825 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 45.126µs I0423 20:43:34.849579 3825 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0423 20:43:34.849588 3825 nad_controller.go:379] [node-nad-controller NAD controller]: started I0423 20:43:34.849592 3825 default_node_network_controller.go:899] Starting the default node network controller I0423 20:43:34.849597 3825 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0423 20:43:34.849669 3825 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 13.718µs I0423 20:43:34.849802 3825 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0423 20:43:34.849811 3825 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0423 20:43:34.849836 3825 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1f330 0xc00032fea0 0xc00032ff10 0xc000340000 0xc000340070 0xc0003400e0 0xc000340150 0xc000b2d770 0xc0003401c0 0xc000340310 0xc000ab8230 0xc0064e5340 0xc000340380 map[0x2ff9080:0xc0040fc120 0x2ff95c0:0xc0040fc360 0x2ffa2e0:0xc0040fc060 0x2ffa580:0xc0040fc0c0 0x2ffad60:0xc0040fc180 0x2ffb000:0xc0040fc1e0 0x2ffb2a0:0xc0040fc300 0x2ffba80:0xc0040fc3c0 0x2ffbd20:0xc0041060c0 0x3026980:0xc002180060 0x3026d20:0xc0025892c0 0x3027800:0xc000c8fb60 0x3029160:0xc001c1c000 0x30298a0:0xc0040fc000 0x302a720:0xc0008ae900] 0xc000b22850 0} I0423 20:43:34.849906 3825 gateway.go:151] Starting gateway service sync I0423 20:43:34.850346 3825 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0423 20:43:34.852763 3825 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0423 20:43:34.877612 3825 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0423 20:43:34.878983 3825 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0423 20:43:34.880247 3825 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0423 20:43:34.914527 3825 gateway.go:168] Gateway service sync done. Time taken: 64.608957ms I0423 20:43:34.915188 3825 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30200 I0423 20:43:34.915435 3825 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1f330 0xc00032fea0 0xc00032ff10 0xc000340000 0xc000340070 0xc0003400e0 0xc000340150 0xc000b2d770 0xc0003401c0 0xc000340310 0xc000ab8230 0xc0064e5340 0xc000340380 map[0x2ff9080:0xc0040fc120 0x2ff95c0:0xc0040fc360 0x2ffa2e0:0xc0040fc060 0x2ffa580:0xc0040fc0c0 0x2ffad60:0xc0040fc180 0x2ffb000:0xc0040fc1e0 0x2ffb2a0:0xc0040fc300 0x2ffba80:0xc0040fc3c0 0x2ffbd20:0xc0041060c0 0x3026980:0xc002180060 0x3026d20:0xc0025892c0 0x3027800:0xc000c8fb60 0x3029160:0xc001c1c000 0x30298a0:0xc0040fc000 0x302a720:0xc0008ae900] 0xc000b22850 0} I0423 20:43:34.916310 3825 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1f330 0xc00032fea0 0xc00032ff10 0xc000340000 0xc000340070 0xc0003400e0 0xc000340150 0xc000b2d770 0xc0003401c0 0xc000340310 0xc000ab8230 0xc0064e5340 0xc000340380 map[0x2ff9080:0xc0040fc120 0x2ff95c0:0xc0040fc360 0x2ffa2e0:0xc0040fc060 0x2ffa580:0xc0040fc0c0 0x2ffad60:0xc0040fc180 0x2ffb000:0xc0040fc1e0 0x2ffb2a0:0xc0040fc300 0x2ffba80:0xc0040fc3c0 0x2ffbd20:0xc0041060c0 0x3026980:0xc002180060 0x3026d20:0xc0025892c0 0x3027800:0xc000c8fb60 0x3029160:0xc001c1c000 0x30298a0:0xc0040fc000 0x302a720:0xc0008ae900] 0xc000b22850 0} I0423 20:43:34.916356 3825 gateway.go:532] Reconciling gateway with updates I0423 20:43:34.921296 3825 gateway.go:339] Spawning Conntrack Rule Check Thread I0423 20:43:34.921317 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:34.921321 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:34.921332 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:34.921333 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:34.921336 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:34.921343 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:34.921356 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:34.921360 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:34.921358 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:34.921364 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:34.921374 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:34.921376 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 20:43:34.921376 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:34.921375 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:34.921387 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:34.921394 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:34.921395 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:34.921405 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:34.921403 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:34.921422 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:34.921424 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 20:43:34.921409 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 20:43:34.921432 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 20:43:34.921437 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 20:43:34.921423 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:34.921442 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:34.921447 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:34.921452 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:34.921454 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:34.921404 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:34.921369 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:34.921461 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:34.921460 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:34.921475 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:34.921484 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:34.921480 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:34.921484 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:34.921484 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:34.921496 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:34.921498 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 20:43:34.921348 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 20:43:34.921506 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 20:43:34.921506 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:34.921513 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 20:43:34.921456 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:34.921520 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:34.921514 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:34.921524 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:34.921532 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:34.921534 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:34.921541 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:34.921539 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:34.921541 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:34.921552 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 20:43:34.921553 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:34.921560 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:34.921561 3825 default_node_network_controller.go:944] Gateway and management port readiness took 71.739528ms I0423 20:43:34.921564 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:34.921583 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:34.921597 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 20:43:34.921605 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:34.921611 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 20:43:34.921619 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 20:43:34.921618 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0423 20:43:34.921651 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 20:43:34.921662 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 20:43:34.921393 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 20:43:34.921665 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 20:43:34.921693 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 20:43:34.921519 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:34.921569 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 20:43:34.921539 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:34.921738 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:34.921427 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:34.921427 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:34.921839 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:34.921437 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:34.921437 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:34.921917 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:34.921443 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 20:43:34.921448 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 20:43:34.921959 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 20:43:34.921455 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:34.921995 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:34.921457 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:34.922072 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:34.921457 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:34.922098 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:34.921461 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:34.921410 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:34.922152 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:34.921462 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:34.922181 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 20:43:34.921456 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:34.922190 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:34.921459 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:34.921488 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:34.921340 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 20:43:34.921492 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:34.922261 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:34.921504 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:34.921501 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:34.922302 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:34.921496 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 20:43:34.922325 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 20:43:34.921520 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:34.922336 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 20:43:34.921520 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:34.922394 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:34.922406 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 20:43:34.921519 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:34.922418 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:34.922431 3825 port_claim.go:71] Svc openshift-ingress/router-default: port 32482 is already open, no action needed I0423 20:43:34.922432 3825 node_ip_handler_linux.go:160] Node IP manager is running I0423 20:43:34.922443 3825 port_claim.go:71] Svc openshift-ingress/router-default: port 32393 is already open, no action needed W0423 20:43:34.960102 3825 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 20:43:34.960316 3825 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0423 20:43:34.960472 3825 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0423 20:43:34.968896 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 20:43:34.968925 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 20:43:34.968934 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 20:43:34.968940 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 20:43:34.968945 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 20:43:34.968951 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 20:43:34.968957 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 20:43:35.008909 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 20:43:35.008943 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 20:43:35.008955 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 20:43:35.008964 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 20:43:35.008975 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 20:43:35.008986 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 20:43:35.008998 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 20:43:35.009008 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 20:43:35.009018 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 20:43:35.009040 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 20:43:35.009058 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 20:43:35.009065 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 20:43:35.009070 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 20:43:35.051780 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 20:43:36.836163 3825 default_node_network_controller.go:1380] Starting node controller node sync W0423 20:43:36.836217 3825 egressip_healthcheck.go:76] Health checking using insecure connection I0423 20:43:36.836266 3825 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0423 20:43:36.849928 3825 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 13.744331ms I0423 20:43:36.850046 3825 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-172.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 20:43:36.850065 3825 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-172.ec2.internal", IP: 10.0.0.172 to PMTUD blocking rules I0423 20:43:36.850046 3825 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-197.ec2.internal", IP: 10.0.1.197 to PMTUD blocking rules I0423 20:43:36.850149 3825 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-197.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0423 20:43:36.862057 3825 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0423 20:43:36.862355 3825 egressservice_node.go:93] Setting up event handlers for Egress Services I0423 20:43:36.862476 3825 egressservice_node.go:183] Starting Egress Services Controller I0423 20:43:36.862496 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0423 20:43:36.862508 3825 shared_informer.go:356] "Caches are synced" controller="egressservices" I0423 20:43:36.862514 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0423 20:43:36.862520 3825 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0423 20:43:36.862527 3825 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0423 20:43:36.862532 3825 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0423 20:43:36.862536 3825 egressservice_node.go:197] Repairing Egress Services I0423 20:43:36.873103 3825 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0423 20:43:36.873115 3825 external_controller.go:278] Starting Admin Policy Based Route Controller I0423 20:43:36.873401 3825 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0423 20:43:36.873414 3825 link_network_manager.go:119] Link manager is running I0423 20:43:36.873423 3825 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0423 20:43:36.873591 3825 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0423 20:43:36.873802 3825 vrf_manager.go:128] VRF manager is running I0423 20:43:36.873888 3825 node_controller_manager.go:437] Removing flows to drop GARP I0423 20:43:36.873897 3825 gateway.go:532] Reconciling gateway with updates I0423 20:43:36.898561 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:36.898574 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:36.898586 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:36.898593 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0423 20:43:36.898592 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:36.898597 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:36.898591 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:36.898609 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0423 20:43:36.898588 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0423 20:43:36.898613 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0423 20:43:36.898619 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:36.898620 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:36.898647 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:36.898652 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:36.898598 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0423 20:43:36.898660 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0423 20:43:36.898664 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0423 20:43:36.898667 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0423 20:43:36.898669 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:36.898574 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:36.898682 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:36.898688 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0423 20:43:36.898684 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:36.898697 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0423 20:43:36.898703 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0423 20:43:36.898703 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:36.898702 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:36.898714 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:36.898654 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0423 20:43:36.898718 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0423 20:43:36.898724 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0423 20:43:36.898727 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:36.898724 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:36.898737 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0423 20:43:36.898737 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:36.898737 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:36.898743 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0423 20:43:36.898751 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0423 20:43:36.898746 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:36.898754 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:36.898758 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0423 20:43:36.898755 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0423 20:43:36.898759 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:36.898764 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0423 20:43:36.898771 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:36.898621 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0423 20:43:36.898774 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0423 20:43:36.898780 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0423 20:43:36.898782 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0423 20:43:36.898784 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0423 20:43:36.898792 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:36.898795 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:36.898796 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0423 20:43:36.898796 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:36.898803 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0423 20:43:36.898695 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:36.898806 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0423 20:43:36.898811 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:36.898817 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0423 20:43:36.898808 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0423 20:43:36.898715 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0423 20:43:36.898751 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0423 20:43:36.898801 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0423 20:43:36.898854 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:36.898782 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:36.898875 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0423 20:43:36.898681 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:36.898887 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:36.898908 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0423 20:43:36.898706 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0423 20:43:36.898895 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0423 20:43:36.898951 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0423 20:43:36.898876 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0423 20:43:36.898972 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:36.898720 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0423 20:43:36.898986 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0423 20:43:36.898689 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0423 20:43:36.898610 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0423 20:43:36.899066 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0423 20:43:36.898604 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:36.899078 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0423 20:43:36.898611 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:36.899122 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0423 20:43:36.898643 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0423 20:43:36.899161 3825 port_claim.go:71] Svc openshift-ingress/router-default: port 32482 is already open, no action needed I0423 20:43:36.898693 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0423 20:43:36.899172 3825 port_claim.go:71] Svc openshift-ingress/router-default: port 32393 is already open, no action needed I0423 20:43:36.898713 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0423 20:43:36.898745 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0423 20:43:36.898746 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0423 20:43:36.898571 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:36.899263 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0423 20:43:36.899269 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0423 20:43:36.898749 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0423 20:43:36.899276 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0423 20:43:36.899279 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0423 20:43:36.898764 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0423 20:43:36.898767 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0423 20:43:36.898762 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:36.899290 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0423 20:43:36.898764 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0423 20:43:36.898774 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:36.899330 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0423 20:43:36.898783 3825 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:36.899383 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0423 20:43:36.899390 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0423 20:43:36.898790 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0423 20:43:36.898810 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:43:36.898823 3825 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0423 20:43:36.941390 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0423 20:43:36.941416 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0423 20:43:36.984744 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0423 20:43:36.984768 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0423 20:43:36.984779 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0423 20:43:36.984790 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0423 20:43:36.984798 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0423 20:43:36.984813 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0423 20:43:36.984826 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0423 20:43:36.984852 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0423 20:43:36.984875 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0423 20:43:36.984882 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0423 20:43:36.984888 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0423 20:43:36.984893 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0423 20:43:36.984902 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0423 20:43:36.984907 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0423 20:43:36.984914 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0423 20:43:37.025798 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0423 20:43:37.025798 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0423 20:43:37.025826 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0423 20:43:37.025835 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0423 20:43:37.025874 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0423 20:43:37.025887 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0423 20:43:37.025896 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0423 20:43:37.025906 3825 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) W0423 20:43:37.272267 3825 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6d44d5bbf5-f4bml IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6d44d5bbf5-f4bml: no pod IPs found W0423 20:43:37.669766 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0423 20:43:38.628814 3825 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-22.ec2.internal I0423 20:43:38.628852 3825 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-22.ec2.internal: 48.107µs I0423 20:43:38.664332 3825 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-587b6c485f-ng4zk] creating logical port openshift-image-registry_image-registry-587b6c485f-ng4zk for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:38.664478 3825 kube.go:256] Updating pod openshift-image-registry/image-registry-587b6c485f-ng4zk I0423 20:43:38.684297 3825 pod.go:62] [openshift-image-registry/image-registry-587b6c485f-ng4zk] pod update took 19.857635ms I0423 20:43:38.684318 3825 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-587b6c485f-ng4zk] addLogicalPort annotation time took 19.883969ms I0423 20:43:38.685506 3825 pods.go:271] [openshift-image-registry/image-registry-587b6c485f-ng4zk] addLogicalPort took 21.190872ms, libovsdb time 874.665µs I0423 20:43:38.706772 3825 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-8tfnb] creating logical port openshift-dns_dns-default-8tfnb for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:38.706887 3825 kube.go:256] Updating pod openshift-dns/dns-default-8tfnb I0423 20:43:38.706949 3825 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-8ps6h] creating logical port openshift-ingress-canary_ingress-canary-8ps6h for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:38.707043 3825 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-8ps6h I0423 20:43:38.728682 3825 pod.go:62] [openshift-ingress-canary/ingress-canary-8ps6h] pod update took 21.654408ms I0423 20:43:38.728703 3825 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-8ps6h] addLogicalPort annotation time took 21.678068ms I0423 20:43:38.729753 3825 pods.go:271] [openshift-ingress-canary/ingress-canary-8ps6h] addLogicalPort took 22.812392ms, libovsdb time 812.961µs I0423 20:43:38.730617 3825 pod.go:62] [openshift-dns/dns-default-8tfnb] pod update took 23.757275ms I0423 20:43:38.730653 3825 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-8tfnb] addLogicalPort annotation time took 23.796887ms I0423 20:43:38.731561 3825 pods.go:271] [openshift-dns/dns-default-8tfnb] addLogicalPort took 24.807296ms, libovsdb time 688.062µs W0423 20:43:38.914980 3825 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 20:43:38.971157 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:43:38.971811 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:43:38.986310 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:43:38.986326 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-bgb65 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:43:38.995074 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c5ccb6fb-bgb65 I0423 20:43:38.995095 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c5ccb6fb-bgb65 I0423 20:43:39.102930 3825 cni.go:452] [openshift-dns/dns-default-8tfnb a7996a08eebd67591e9aebc8db222986b72ec5a9443f246bdf82dd5956ff98f1 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-8tfnb a7996a08eebd67591e9aebc8db222986b72ec5a9443f246bdf82dd5956ff98f1 network default NAD default NAD key ] I0423 20:43:39.105787 3825 cni.go:452] [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key ] I0423 20:43:39.129437 3825 cni.go:452] [openshift-ingress-canary/ingress-canary-8ps6h 8d21ce6706b8a1e641890d121151947a64e4700fcf93b8554f2df1336364f16d network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-8ps6h 8d21ce6706b8a1e641890d121151947a64e4700fcf93b8554f2df1336364f16d network default NAD default NAD key ] I0423 20:43:39.144247 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-8tfnb, hostIfaceName: a7996a08eebd675, network: default, NAD default, SandboxID: "a7996a08eebd67591e9aebc8db222986b72ec5a9443f246bdf82dd5956ff98f1", PCI device ID: , UID: "501c41d1-2590-4a70-9b24-545dc59fb6de", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0423 20:43:39.144253 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-587b6c485f-ng4zk, hostIfaceName: 95912db9915f3b0, network: default, NAD default, SandboxID: "95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015", PCI device ID: , UID: "47a2a1cb-a3f6-4229-9cff-f64703419d4b", MAC: 0a:58:0a:82:00:05, IPs: [10.130.0.5/23] I0423 20:43:39.144336 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-8ps6h, hostIfaceName: 8d21ce6706b8a1e, network: default, NAD default, SandboxID: "8d21ce6706b8a1e641890d121151947a64e4700fcf93b8554f2df1336364f16d", PCI device ID: , UID: "2bd25915-2f7e-456e-b1b1-1fe2246bdb0c", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0423 20:43:39.196345 3825 cni.go:473] [openshift-dns/dns-default-8tfnb a7996a08eebd67591e9aebc8db222986b72ec5a9443f246bdf82dd5956ff98f1 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-8tfnb a7996a08eebd67591e9aebc8db222986b72ec5a9443f246bdf82dd5956ff98f1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:50:02:29:2a:d3\",\"name\":\"a7996a08eebd675\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9f30569-ebb3-4eef-8cbf-36b6f6b8e091\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:43:39.201228 3825 cni.go:473] [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:51:e1:ba:cb:6c\",\"name\":\"95912db9915f3b0\"},{\"mac\":\"0a:58:0a:82:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb1774fb-1dec-4c8f-ba3b-c76999122312\"}],\"ips\":[{\"address\":\"10.130.0.5/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:43:39.208538 3825 cni.go:473] [openshift-ingress-canary/ingress-canary-8ps6h 8d21ce6706b8a1e641890d121151947a64e4700fcf93b8554f2df1336364f16d network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-8ps6h 8d21ce6706b8a1e641890d121151947a64e4700fcf93b8554f2df1336364f16d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:a2:c8:fb:f0:c6\",\"name\":\"8d21ce6706b8a1e\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/56ec98d2-04a8-4734-ae67-b96d5521dd69\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:43:41.800109 3825 cni.go:452] [openshift-multus/network-metrics-daemon-khgmn 9dd1bdd4d9f05c9ed80c211d6e2743ede1794832ddcf91fbc5afa37e83c26034 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-khgmn 9dd1bdd4d9f05c9ed80c211d6e2743ede1794832ddcf91fbc5afa37e83c26034 network default NAD default NAD key ] I0423 20:43:41.812997 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-khgmn, hostIfaceName: 9dd1bdd4d9f05c9, network: default, NAD default, SandboxID: "9dd1bdd4d9f05c9ed80c211d6e2743ede1794832ddcf91fbc5afa37e83c26034", PCI device ID: , UID: "00eb364c-9243-439f-b5cf-fc5021682c22", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0423 20:43:41.848408 3825 cni.go:452] [openshift-network-diagnostics/network-check-target-rw6vl 2c7dfcc13a1c856a46b59912cc517b26caa46838eaa84a61b3d29434f347fc26 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-rw6vl 2c7dfcc13a1c856a46b59912cc517b26caa46838eaa84a61b3d29434f347fc26 network default NAD default NAD key ] I0423 20:43:41.860076 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-rw6vl, hostIfaceName: 2c7dfcc13a1c856, network: default, NAD default, SandboxID: "2c7dfcc13a1c856a46b59912cc517b26caa46838eaa84a61b3d29434f347fc26", PCI device ID: , UID: "cb54ace5-78fd-47a3-9231-0c77e0512f5d", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0423 20:43:41.869004 3825 cni.go:473] [openshift-multus/network-metrics-daemon-khgmn 9dd1bdd4d9f05c9ed80c211d6e2743ede1794832ddcf91fbc5afa37e83c26034 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-khgmn 9dd1bdd4d9f05c9ed80c211d6e2743ede1794832ddcf91fbc5afa37e83c26034 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:04:f6:b1:95:e7\",\"name\":\"9dd1bdd4d9f05c9\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d7566ba3-921d-44e8-9141-55eb4e64aa46\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:43:41.912174 3825 cni.go:473] [openshift-network-diagnostics/network-check-target-rw6vl 2c7dfcc13a1c856a46b59912cc517b26caa46838eaa84a61b3d29434f347fc26 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-rw6vl 2c7dfcc13a1c856a46b59912cc517b26caa46838eaa84a61b3d29434f347fc26 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:fd:6f:aa:bb:da\",\"name\":\"2c7dfcc13a1c856\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67fb361f-bc3e-4aa0-9ef0-0fa8d0f916a2\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:43:45.769328 3825 default_network_controller.go:1025] Node "ip-10-0-0-172.ec2.internal" in remote zone "ip-10-0-0-172.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0423 20:43:45.770364 3825 zone_ic_handler.go:195] Time taken to create transit switch: 279.387µs I0423 20:43:45.770379 3825 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-172.ec2.internal for the network default I0423 20:43:45.771289 3825 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-172.ec2.internal" on network "default" took: 1.221707ms W0423 20:43:55.456499 3825 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:43:55.459393 3825 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0423 20:43:55.551907 3825 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-52kqf] creating logical port openshift-insights_insights-runtime-extractor-52kqf for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:55.552063 3825 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-52kqf W0423 20:43:55.619110 3825 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 20:43:55.638227 3825 pod.go:62] [openshift-insights/insights-runtime-extractor-52kqf] pod update took 86.204545ms I0423 20:43:55.638248 3825 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-52kqf] addLogicalPort annotation time took 86.229883ms I0423 20:43:55.639393 3825 pods.go:271] [openshift-insights/insights-runtime-extractor-52kqf] addLogicalPort took 87.504887ms, libovsdb time 878.467µs W0423 20:43:55.660445 3825 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0423 20:43:55.913372 3825 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-mhgds] creating logical port openshift-console_downloads-5c54f4944f-mhgds for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:55.913470 3825 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-mhgds I0423 20:43:55.951380 3825 pod.go:62] [openshift-console/downloads-5c54f4944f-mhgds] pod update took 37.936155ms I0423 20:43:55.951410 3825 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-mhgds] addLogicalPort annotation time took 37.971101ms I0423 20:43:55.951705 3825 cni.go:452] [openshift-insights/insights-runtime-extractor-52kqf bdb48a64dc1994036b0c5a619fb98f662d674f5fc195c7c83f74c2c5213b8981 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-52kqf bdb48a64dc1994036b0c5a619fb98f662d674f5fc195c7c83f74c2c5213b8981 network default NAD default NAD key ] I0423 20:43:55.952660 3825 pods.go:271] [openshift-console/downloads-5c54f4944f-mhgds] addLogicalPort took 39.303344ms, libovsdb time 904.151µs I0423 20:43:55.962250 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-52kqf, hostIfaceName: bdb48a64dc19940, network: default, NAD default, SandboxID: "bdb48a64dc1994036b0c5a619fb98f662d674f5fc195c7c83f74c2c5213b8981", PCI device ID: , UID: "50dca26a-4494-4019-ad10-6c1d5036bbb3", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0423 20:43:56.010953 3825 cni.go:473] [openshift-insights/insights-runtime-extractor-52kqf bdb48a64dc1994036b0c5a619fb98f662d674f5fc195c7c83f74c2c5213b8981 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-52kqf bdb48a64dc1994036b0c5a619fb98f662d674f5fc195c7c83f74c2c5213b8981 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:c2:5b:20:57:cc\",\"name\":\"bdb48a64dc19940\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e944f7af-488f-40f4-965e-68f55d43b551\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:43:56.293370 3825 cni.go:452] [openshift-console/downloads-5c54f4944f-mhgds fb53caf448a378a7d465fa32ddefa4777391cd929c6cb8df429f0e1c690da1a4 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-mhgds fb53caf448a378a7d465fa32ddefa4777391cd929c6cb8df429f0e1c690da1a4 network default NAD default NAD key ] I0423 20:43:56.305301 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-mhgds, hostIfaceName: fb53caf448a378a, network: default, NAD default, SandboxID: "fb53caf448a378a7d465fa32ddefa4777391cd929c6cb8df429f0e1c690da1a4", PCI device ID: , UID: "4675c075-8e90-498b-b8f7-678f1822bf6e", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0423 20:43:56.350680 3825 cni.go:473] [openshift-console/downloads-5c54f4944f-mhgds fb53caf448a378a7d465fa32ddefa4777391cd929c6cb8df429f0e1c690da1a4 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-mhgds fb53caf448a378a7d465fa32ddefa4777391cd929c6cb8df429f0e1c690da1a4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:3b:2e:31:c8:1b\",\"name\":\"fb53caf448a378a\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5170fc4-a7c2-477c-b0f7-1930e374e60e\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:43:56.993843 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:56.993909 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:56.994050 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:56.994070 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:56.994078 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.039242 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.039299 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.042689 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.042704 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.042710 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.074767 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.074786 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0423 20:43:57.076549 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.076562 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0423 20:43:57.076567 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default I0423 20:43:59.926309 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-64sk8 for pod on switch ip-10-0-2-22.ec2.internal I0423 20:43:59.926460 3825 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8 I0423 20:43:59.947048 3825 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8] pod update took 20.616382ms I0423 20:43:59.947075 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8] addLogicalPort annotation time took 20.651337ms I0423 20:43:59.948408 3825 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8] addLogicalPort took 22.114007ms, libovsdb time 941.097µs I0423 20:44:00.913051 3825 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8 40e2f100853fbc3e59e9ecfa41d9e2f7509e3c1c3067d9e175ce596e707e5166 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8 40e2f100853fbc3e59e9ecfa41d9e2f7509e3c1c3067d9e175ce596e707e5166 network default NAD default NAD key ] I0423 20:44:00.927679 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-64sk8, hostIfaceName: 40e2f100853fbc3, network: default, NAD default, SandboxID: "40e2f100853fbc3e59e9ecfa41d9e2f7509e3c1c3067d9e175ce596e707e5166", PCI device ID: , UID: "70531f23-caa2-4683-a3bc-c8163e0dae0e", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0423 20:44:00.980731 3825 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8 40e2f100853fbc3e59e9ecfa41d9e2f7509e3c1c3067d9e175ce596e707e5166 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-64sk8 40e2f100853fbc3e59e9ecfa41d9e2f7509e3c1c3067d9e175ce596e707e5166 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:63:08:c5:2f:3f\",\"name\":\"40e2f100853fbc3\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea6ec3eb-428c-4a94-ab90-6dd67e6144f6\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:44:00.993044 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:00.993073 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:01.952739 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:01.952762 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:03.005325 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0423 20:44:03.032913 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0423 20:44:03.185300 3825 base_network_controller_pods.go:487] [default/openshift-console/console-7b549f9c9d-7l4cm] creating logical port openshift-console_console-7b549f9c9d-7l4cm for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:03.185429 3825 kube.go:256] Updating pod openshift-console/console-7b549f9c9d-7l4cm I0423 20:44:03.201867 3825 pod.go:62] [openshift-console/console-7b549f9c9d-7l4cm] pod update took 16.47488ms I0423 20:44:03.201951 3825 base_network_controller_pods.go:951] [default/openshift-console/console-7b549f9c9d-7l4cm] addLogicalPort annotation time took 16.563029ms I0423 20:44:03.203303 3825 pods.go:271] [openshift-console/console-7b549f9c9d-7l4cm] addLogicalPort took 18.023854ms, libovsdb time 1.067972ms I0423 20:44:03.549345 3825 cni.go:452] [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key ] I0423 20:44:03.560476 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7b549f9c9d-7l4cm, hostIfaceName: 407a73c27b56a31, network: default, NAD default, SandboxID: "407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe", PCI device ID: , UID: "ee18b187-279b-422e-bf31-257e546adf8b", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0423 20:44:03.608671 3825 cni.go:473] [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:5d:ef:25:2e:17\",\"name\":\"407a73c27b56a31\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/14806237-112f-4a1e-8370-9ac64b26fe59\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:44:04.015163 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-l27nf] creating logical port openshift-monitoring_prometheus-operator-76b589d668-l27nf for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:04.015285 3825 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-l27nf I0423 20:44:04.032058 3825 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-l27nf] pod update took 16.798738ms I0423 20:44:04.032082 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-l27nf] addLogicalPort annotation time took 16.826497ms I0423 20:44:04.033225 3825 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-l27nf] addLogicalPort took 18.078475ms, libovsdb time 867.007µs I0423 20:44:04.373585 3825 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-l27nf 99cfcede191bd550c74898d855f9b7b4ec0a29e1c1530f7a9e3f3f97d0f4b811 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-l27nf 99cfcede191bd550c74898d855f9b7b4ec0a29e1c1530f7a9e3f3f97d0f4b811 network default NAD default NAD key ] I0423 20:44:04.385768 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-l27nf, hostIfaceName: 99cfcede191bd55, network: default, NAD default, SandboxID: "99cfcede191bd550c74898d855f9b7b4ec0a29e1c1530f7a9e3f3f97d0f4b811", PCI device ID: , UID: "9369c28d-ffdd-4ec2-a70c-9863add11058", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0423 20:44:04.438060 3825 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-l27nf 99cfcede191bd550c74898d855f9b7b4ec0a29e1c1530f7a9e3f3f97d0f4b811 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-l27nf 99cfcede191bd550c74898d855f9b7b4ec0a29e1c1530f7a9e3f3f97d0f4b811 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:80:3b:f1:58:d6\",\"name\":\"99cfcede191bd55\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7cefbe2a-a882-479a-a99c-dd1fc57bef88\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:44:07.040320 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 20:44:08.055462 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0423 20:44:08.097209 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0423 20:44:10.573991 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-pp9zv for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:10.574104 3825 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv I0423 20:44:10.591909 3825 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv] pod update took 17.830013ms I0423 20:44:10.591988 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv] addLogicalPort annotation time took 17.914647ms I0423 20:44:10.593155 3825 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv] addLogicalPort took 19.185ms, libovsdb time 875.165µs I0423 20:44:10.601904 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts] creating logical port openshift-monitoring_kube-state-metrics-6dfffbb599-lq2ts for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:10.602042 3825 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts I0423 20:44:10.622474 3825 pod.go:62] [openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts] pod update took 20.459707ms I0423 20:44:10.622501 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts] addLogicalPort annotation time took 20.490521ms I0423 20:44:10.623901 3825 pods.go:271] [openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts] addLogicalPort took 22.01411ms, libovsdb time 971.525µs I0423 20:44:11.842400 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:11.842547 3825 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0423 20:44:11.868075 3825 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 25.568951ms I0423 20:44:11.868100 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 25.599381ms I0423 20:44:11.869315 3825 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 26.931025ms, libovsdb time 901.23µs I0423 20:44:13.646718 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq] creating logical port openshift-monitoring_thanos-querier-6bcfbb6b8c-fq4gq for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:13.646979 3825 kube.go:256] Updating pod openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq I0423 20:44:13.669106 3825 pod.go:62] [openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq] pod update took 22.191265ms I0423 20:44:13.669137 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq] addLogicalPort annotation time took 22.225903ms I0423 20:44:13.670554 3825 pods.go:271] [openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq] addLogicalPort took 23.858455ms, libovsdb time 1.001382ms W0423 20:44:14.085233 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:14.085255 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:14.175424 3825 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:14.175441 3825 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:14.175447 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:15.188890 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:15.188915 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:15.249698 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:15.249728 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 20:44:15.269002 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-9bsl8 for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:15.269118 3825 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8 I0423 20:44:15.287421 3825 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8] pod update took 18.335868ms I0423 20:44:15.287511 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8] addLogicalPort annotation time took 18.429164ms I0423 20:44:15.288877 3825 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8] addLogicalPort took 19.892687ms, libovsdb time 854.069µs I0423 20:44:16.501595 3825 cni.go:452] [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key ] I0423 20:44:16.502060 3825 cni.go:452] [openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq 63f257f7035f64ea930e47e2a73c867d0bfb0d146fc171ee0b769c10f842c99d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq 63f257f7035f64ea930e47e2a73c867d0bfb0d146fc171ee0b769c10f842c99d network default NAD default NAD key ] I0423 20:44:16.502096 3825 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8 01e502176cf6b9e608154d496785d8b46e7e91dd4a9b1b9c93e08df488183ddc network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8 01e502176cf6b9e608154d496785d8b46e7e91dd4a9b1b9c93e08df488183ddc network default NAD default NAD key ] I0423 20:44:16.502109 3825 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv 16de4a29f0e8942fea0b4ced968c3e953b8413b30086ff1f8ef9d1fbac5e7440 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv 16de4a29f0e8942fea0b4ced968c3e953b8413b30086ff1f8ef9d1fbac5e7440 network default NAD default NAD key ] I0423 20:44:16.503301 3825 cni.go:452] [openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts 5a6a18be4d3a5ee764abbf0a49877eaf9a45c5d2d833608b7b0fdf141189ec9c network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts 5a6a18be4d3a5ee764abbf0a49877eaf9a45c5d2d833608b7b0fdf141189ec9c network default NAD default NAD key ] I0423 20:44:16.520089 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-9bsl8, hostIfaceName: 01e502176cf6b9e, network: default, NAD default, SandboxID: "01e502176cf6b9e608154d496785d8b46e7e91dd4a9b1b9c93e08df488183ddc", PCI device ID: , UID: "0d508aab-8846-4774-be72-f67abcc7f975", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0423 20:44:16.520101 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 53c3055b928dad9, network: default, NAD default, SandboxID: "53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef", PCI device ID: , UID: "d1101f9d-df3f-4363-92ca-9e54e7a42ef8", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0423 20:44:16.520583 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-pp9zv, hostIfaceName: 16de4a29f0e8942, network: default, NAD default, SandboxID: "16de4a29f0e8942fea0b4ced968c3e953b8413b30086ff1f8ef9d1fbac5e7440", PCI device ID: , UID: "e9c5b554-97b1-4d1d-b04b-c788540cc88b", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0423 20:44:16.521500 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-6bcfbb6b8c-fq4gq, hostIfaceName: 63f257f7035f64e, network: default, NAD default, SandboxID: "63f257f7035f64ea930e47e2a73c867d0bfb0d146fc171ee0b769c10f842c99d", PCI device ID: , UID: "9d35d790-0b46-4be6-99fd-834c0c9cbb61", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0423 20:44:16.521820 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-6dfffbb599-lq2ts, hostIfaceName: 5a6a18be4d3a5ee, network: default, NAD default, SandboxID: "5a6a18be4d3a5ee764abbf0a49877eaf9a45c5d2d833608b7b0fdf141189ec9c", PCI device ID: , UID: "b2f9c149-d6c2-493f-81d8-aad77617da52", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0423 20:44:16.576437 3825 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv 16de4a29f0e8942fea0b4ced968c3e953b8413b30086ff1f8ef9d1fbac5e7440 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-pp9zv 16de4a29f0e8942fea0b4ced968c3e953b8413b30086ff1f8ef9d1fbac5e7440 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:e1:c2:55:63:a1\",\"name\":\"16de4a29f0e8942\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85418561-45ea-426d-9250-7aac9b3a3830\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:44:16.594114 3825 cni.go:473] [openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq 63f257f7035f64ea930e47e2a73c867d0bfb0d146fc171ee0b769c10f842c99d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-6bcfbb6b8c-fq4gq 63f257f7035f64ea930e47e2a73c867d0bfb0d146fc171ee0b769c10f842c99d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:72:78:5a:28:25\",\"name\":\"63f257f7035f64e\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/94442234-aca7-44e5-b2d1-310f5a193f76\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:44:16.820265 3825 cni.go:473] [openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts 5a6a18be4d3a5ee764abbf0a49877eaf9a45c5d2d833608b7b0fdf141189ec9c network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-lq2ts 5a6a18be4d3a5ee764abbf0a49877eaf9a45c5d2d833608b7b0fdf141189ec9c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:50:f1:e2:11:e4\",\"name\":\"5a6a18be4d3a5ee\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d9ad3a50-dad5-4d21-8229-c715ab8d44d5\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:44:16.820299 3825 cni.go:473] [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:dc:3d:d7:99:f9\",\"name\":\"53c3055b928dad9\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/317f524d-5b7d-4134-9c3c-f1c726cd65e4\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:44:16.820332 3825 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8 01e502176cf6b9e608154d496785d8b46e7e91dd4a9b1b9c93e08df488183ddc network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-9bsl8 01e502176cf6b9e608154d496785d8b46e7e91dd4a9b1b9c93e08df488183ddc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:30:f3:aa:d0:57\",\"name\":\"01e502176cf6b9e\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/afbdf319-9fdf-4f50-9594-5c1d8ca7d7b1\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:44:16.969547 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0423 20:44:17.006307 3825 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-7cc8fc68d5-d94fz IP address from the namespace address-set, err: pod openshift-console/console-7cc8fc68d5-d94fz: no pod IPs found W0423 20:44:17.087748 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0423 20:44:17.101763 3825 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0423 20:44:17.127266 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0423 20:44:17.127365 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0423 20:44:17.239486 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:17.239657 3825 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0423 20:44:17.267790 3825 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 28.210932ms I0423 20:44:17.267875 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 28.300817ms I0423 20:44:17.269416 3825 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 29.948004ms, libovsdb time 1.08563ms I0423 20:44:17.635563 3825 cni.go:452] [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key ] I0423 20:44:17.647863 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 866adc235f3e09d, network: default, NAD default, SandboxID: "866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a", PCI device ID: , UID: "216f4c68-9006-4695-af51-89bd62269fed", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0423 20:44:17.711664 3825 cni.go:473] [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:3e:6c:4e:6b:00\",\"name\":\"866adc235f3e09d\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/422bba6d-5f4d-4fd7-86c5-08ec87da8d47\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:44:18.108293 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0423 20:44:18.218455 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 20:44:19.096875 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0423 20:44:19.174428 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0423 20:44:19.196850 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:44:19.196872 3825 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:44:19.229450 3825 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:19.229471 3825 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:19.229477 3825 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:19.229486 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:20.351180 3825 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:20.351208 3825 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:20.351216 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:20.525994 3825 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:44:20.526017 3825 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0423 20:44:20.783409 3825 cni.go:452] [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key ] I0423 20:44:20.872937 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.873724 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.884166 3825 cni.go:473] [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-587b6c485f-ng4zk 95912db9915f3b0124c316ba7162e29f1d195af0076f62c077642ae7222df015 network default NAD default NAD key default], result "{}", err I0423 20:44:20.987268 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.987384 3825 pods.go:176] Deleting pod: openshift-image-registry/image-registry-587b6c485f-ng4zk I0423 20:44:20.989619 3825 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-587b6c485f-ng4zk, ips: 10.130.0.5 I0423 20:44:20.989741 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:20.999211 3825 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-66bc67cd8c-z5ldp] creating logical port openshift-image-registry_image-registry-66bc67cd8c-z5ldp for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:20.999337 3825 kube.go:256] Updating pod openshift-image-registry/image-registry-66bc67cd8c-z5ldp I0423 20:44:21.015595 3825 pod.go:62] [openshift-image-registry/image-registry-66bc67cd8c-z5ldp] pod update took 16.292529ms I0423 20:44:21.015649 3825 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-66bc67cd8c-z5ldp] addLogicalPort annotation time took 16.33292ms I0423 20:44:21.017360 3825 pods.go:271] [openshift-image-registry/image-registry-66bc67cd8c-z5ldp] addLogicalPort took 18.167351ms, libovsdb time 1.276195ms I0423 20:44:21.119977 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.120069 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6c5ccb6fb-2t4mj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.127568 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6c5ccb6fb-2t4mj I0423 20:44:21.127600 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6c5ccb6fb-2t4mj I0423 20:44:21.180372 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.180426 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-ng4zk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:21.189931 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-587b6c485f-ng4zk I0423 20:44:21.190034 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-587b6c485f-ng4zk I0423 20:44:21.437501 3825 cni.go:452] [openshift-image-registry/image-registry-66bc67cd8c-z5ldp 6bd3e0c9eafc3684c04e789a53c8b5509b7ee991a7f3f63c3f861840028d9e52 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-66bc67cd8c-z5ldp 6bd3e0c9eafc3684c04e789a53c8b5509b7ee991a7f3f63c3f861840028d9e52 network default NAD default NAD key ] I0423 20:44:21.449467 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-66bc67cd8c-z5ldp, hostIfaceName: 6bd3e0c9eafc368, network: default, NAD default, SandboxID: "6bd3e0c9eafc3684c04e789a53c8b5509b7ee991a7f3f63c3f861840028d9e52", PCI device ID: , UID: "579fbdbc-0691-46ca-8ae4-14f598a2caa3", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0423 20:44:21.515429 3825 cni.go:473] [openshift-image-registry/image-registry-66bc67cd8c-z5ldp 6bd3e0c9eafc3684c04e789a53c8b5509b7ee991a7f3f63c3f861840028d9e52 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-66bc67cd8c-z5ldp 6bd3e0c9eafc3684c04e789a53c8b5509b7ee991a7f3f63c3f861840028d9e52 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:a9:6a:92:ce:79\",\"name\":\"6bd3e0c9eafc368\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af82ada3-d341-431c-83c9-6396259ed1ce\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:44:28.168363 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:28.168395 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:28.168405 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0423 20:44:28.168413 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0423 20:44:33.180830 3825 cni.go:452] [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key ] I0423 20:44:33.257289 3825 cni.go:473] [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7b549f9c9d-7l4cm 407a73c27b56a31771644409f92b897a416d0c2fb588704ae1c03a287c976afe network default NAD default NAD key default], result "{}", err I0423 20:44:33.328617 3825 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:33.328680 3825 pods.go:176] Deleting pod: openshift-console/console-7b549f9c9d-7l4cm I0423 20:44:33.329891 3825 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7b549f9c9d-7l4cm, ips: 10.130.0.11 I0423 20:44:33.329921 3825 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:33.341368 3825 base_network_controller_pods.go:487] [default/openshift-console/console-665c78db9-pc9gx] creating logical port openshift-console_console-665c78db9-pc9gx for pod on switch ip-10-0-2-22.ec2.internal I0423 20:44:33.341499 3825 kube.go:256] Updating pod openshift-console/console-665c78db9-pc9gx I0423 20:44:33.355798 3825 pod.go:62] [openshift-console/console-665c78db9-pc9gx] pod update took 14.324581ms I0423 20:44:33.355820 3825 base_network_controller_pods.go:951] [default/openshift-console/console-665c78db9-pc9gx] addLogicalPort annotation time took 14.349324ms I0423 20:44:33.357028 3825 pods.go:271] [openshift-console/console-665c78db9-pc9gx] addLogicalPort took 15.675988ms, libovsdb time 893.818µs I0423 20:44:33.752734 3825 cni.go:452] [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key ] I0423 20:44:33.762121 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-665c78db9-pc9gx, hostIfaceName: b33e50957e096a6, network: default, NAD default, SandboxID: "b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab", PCI device ID: , UID: "40210b4c-0a3a-4dce-bec3-15344f4266e9", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0423 20:44:33.812292 3825 cni.go:473] [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:50:1b:6b:0e:d9\",\"name\":\"b33e50957e096a6\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cda44571-ebbc-49a5-84f5-5b1c2dde9882\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:44:34.227914 3825 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:34.227939 3825 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-7l4cm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:34.235155 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b549f9c9d-7l4cm I0423 20:44:34.235181 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b549f9c9d-7l4cm I0423 20:44:41.377859 3825 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:41.378433 3825 obj_retry.go:498] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:42.205614 3825 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:42.205664 3825 obj_retry.go:492] Detected object openshift-console/console-7b549f9c9d-cmgt9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:44:42.208599 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b549f9c9d-cmgt9 I0423 20:44:42.208622 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b549f9c9d-cmgt9 W0423 20:44:42.285055 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:42.285081 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:42.285090 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:42.285102 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:42.285108 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:42.285112 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:43.132014 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:43.132041 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:43.165514 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0423 20:44:43.292139 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:43.292161 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:43.292167 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:44:44.257777 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0423 20:44:44.287111 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0423 20:44:45.207245 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0423 20:44:59.415660 3825 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:44:59.416220 3825 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0423 20:45:00.272273 3825 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0423 20:45:00.439543 3825 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:00.439576 3825 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616280-2tvcw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:09.471807 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:09.472377 3825 obj_retry.go:498] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:09.609989 3825 obj_retry.go:498] Detected object openshift-console/console-7cc8fc68d5-s269m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:09.610600 3825 obj_retry.go:498] Detected object openshift-console/console-7cc8fc68d5-s269m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:10.405769 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.405801 3825 obj_retry.go:492] Detected object openshift-image-registry/image-registry-587b6c485f-rvc8z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.412336 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-587b6c485f-rvc8z I0423 20:45:10.412354 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-587b6c485f-rvc8z I0423 20:45:10.431225 3825 obj_retry.go:492] Detected object openshift-console/console-7cc8fc68d5-s269m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.431246 3825 obj_retry.go:492] Detected object openshift-console/console-7cc8fc68d5-s269m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:10.437819 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7cc8fc68d5-s269m I0423 20:45:10.437840 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7cc8fc68d5-s269m I0423 20:45:13.338370 3825 obj_retry.go:498] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:13.339005 3825 obj_retry.go:498] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:13.717966 3825 obj_retry.go:492] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:13.717993 3825 obj_retry.go:492] Detected object openshift-ingress/router-default-7bcc69756c-9bl9j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:13.724853 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7bcc69756c-9bl9j I0423 20:45:13.724879 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7bcc69756c-9bl9j W0423 20:45:18.440987 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:45:18.442014 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0423 20:45:32.716647 3825 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:32.717258 3825 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:33.510939 3825 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:33.510964 3825 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:33.519420 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0423 20:45:33.519448 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0423 20:45:37.123345 3825 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:37.123970 3825 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:45:37.552501 3825 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:37.552526 3825 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:45:37.560272 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0423 20:45:37.560303 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0423 20:46:05.050755 3825 cni.go:452] [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key ] I0423 20:46:05.122486 3825 cni.go:473] [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-665c78db9-pc9gx b33e50957e096a6010783101cc2a278315cb55d53ef2cd08a723d3d3a904acab network default NAD default NAD key default], result "{}", err I0423 20:46:05.147218 3825 cni.go:452] [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key ] I0423 20:46:05.213867 3825 obj_retry.go:498] Detected object openshift-console/console-665c78db9-pc9gx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.213902 3825 pods.go:176] Deleting pod: openshift-console/console-665c78db9-pc9gx I0423 20:46:05.215148 3825 pods.go:236] Attempting to release IPs for pod: openshift-console/console-665c78db9-pc9gx, ips: 10.130.0.20 I0423 20:46:05.215177 3825 obj_retry.go:498] Detected object openshift-console/console-665c78db9-pc9gx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.224120 3825 cni.go:473] [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 53c3055b928dad92942324b988d2719d96d9a4d3a72fdcc4fce422075b68e2ef network default NAD default NAD key default], result "{}", err I0423 20:46:05.235108 3825 base_network_controller_pods.go:487] [default/openshift-console/console-86f657c86d-sbb9l] creating logical port openshift-console_console-86f657c86d-sbb9l for pod on switch ip-10-0-2-22.ec2.internal I0423 20:46:05.235274 3825 kube.go:256] Updating pod openshift-console/console-86f657c86d-sbb9l I0423 20:46:05.260703 3825 pod.go:62] [openshift-console/console-86f657c86d-sbb9l] pod update took 25.466258ms I0423 20:46:05.260731 3825 base_network_controller_pods.go:951] [default/openshift-console/console-86f657c86d-sbb9l] addLogicalPort annotation time took 25.502867ms I0423 20:46:05.262026 3825 pods.go:271] [openshift-console/console-86f657c86d-sbb9l] addLogicalPort took 26.935123ms, libovsdb time 948.932µs I0423 20:46:05.324696 3825 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.324813 3825 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0423 20:46:05.326299 3825 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.130.0.15 I0423 20:46:05.326332 3825 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:05.591177 3825 obj_retry.go:492] Detected object openshift-console/console-665c78db9-pc9gx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.591209 3825 obj_retry.go:492] Detected object openshift-console/console-665c78db9-pc9gx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.598433 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-665c78db9-pc9gx I0423 20:46:05.598459 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-665c78db9-pc9gx I0423 20:46:05.600349 3825 cni.go:452] [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key ] I0423 20:46:05.610741 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-86f657c86d-sbb9l, hostIfaceName: 10004cf56ab1f92, network: default, NAD default, SandboxID: "10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d", PCI device ID: , UID: "93f557ef-c905-49b0-a241-e31653d99d7d", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0423 20:46:05.620429 3825 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.620463 3825 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:05.628649 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0423 20:46:05.628736 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0423 20:46:05.662622 3825 cni.go:473] [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:80:df:e5:97:62\",\"name\":\"10004cf56ab1f92\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/945bfcf8-4f96-4596-b711-374f744ded42\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:46:05.668980 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-22.ec2.internal I0423 20:46:05.669110 3825 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0423 20:46:05.699266 3825 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 30.205377ms I0423 20:46:05.699295 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 30.239637ms I0423 20:46:05.700607 3825 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 31.645884ms, libovsdb time 932.354µs I0423 20:46:06.028981 3825 cni.go:452] [openshift-monitoring/alertmanager-main-0 00671ee7a5f2c794f8054ec705a46fa153e4f8da873b8f32388d5541b3d0b2a1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 00671ee7a5f2c794f8054ec705a46fa153e4f8da873b8f32388d5541b3d0b2a1 network default NAD default NAD key ] I0423 20:46:06.038888 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 00671ee7a5f2c79, network: default, NAD default, SandboxID: "00671ee7a5f2c794f8054ec705a46fa153e4f8da873b8f32388d5541b3d0b2a1", PCI device ID: , UID: "aa3c70e9-0c27-40bd-a735-0b17582a8258", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0423 20:46:06.086659 3825 cni.go:473] [openshift-monitoring/alertmanager-main-0 00671ee7a5f2c794f8054ec705a46fa153e4f8da873b8f32388d5541b3d0b2a1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 00671ee7a5f2c794f8054ec705a46fa153e4f8da873b8f32388d5541b3d0b2a1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:1c:05:5e:a3:7d\",\"name\":\"00671ee7a5f2c79\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/89fa536a-631e-4baa-88f7-956f0c4b9899\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:46:10.669371 3825 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.669907 3825 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.669940 3825 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.669955 3825 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0423 20:46:10.669969 3825 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0423 20:46:10.669974 3825 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0423 20:46:11.695958 3825 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:11.695983 3825 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:11.696003 3825 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:11.696010 3825 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390p62d6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:14.232363 3825 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0423 20:46:14.244695 3825 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0423 20:46:15.654342 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 20:46:15.969134 3825 obj_retry.go:498] Detected object openshift-console/console-665c78db9-wswd9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:15.969663 3825 obj_retry.go:498] Detected object openshift-console/console-665c78db9-wswd9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:16.566306 3825 obj_retry.go:492] Detected object openshift-console/console-665c78db9-wswd9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:16.566330 3825 obj_retry.go:492] Detected object openshift-console/console-665c78db9-wswd9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:16.572704 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-665c78db9-wswd9 I0423 20:46:16.572738 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-665c78db9-wswd9 W0423 20:46:36.092167 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:46:36.092191 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0423 20:46:36.092197 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0423 20:46:40.035566 3825 cni.go:452] [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key ] I0423 20:46:40.126941 3825 cni.go:473] [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 866adc235f3e09d82c55badf219427626fea0ca8dae36d1efcdb49beb70cae9a network default NAD default NAD key default], result "{}", err I0423 20:46:40.205699 3825 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:40.205749 3825 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0423 20:46:40.207148 3825 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.130.0.18 I0423 20:46:40.207184 3825 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:46:40.744421 3825 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:40.744474 3825 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:46:40.752986 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0423 20:46:40.753016 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0423 20:46:40.799232 3825 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-22.ec2.internal I0423 20:46:40.799377 3825 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0423 20:46:40.824698 3825 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 25.355442ms I0423 20:46:40.824750 3825 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 25.412297ms I0423 20:46:40.826147 3825 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 26.935097ms, libovsdb time 1.051091ms I0423 20:47:08.783566 3825 cni.go:452] [openshift-monitoring/prometheus-k8s-0 341bc9c56078586455583851873fbd264b8c2b6c7077c36b033c3d82a38d32d5 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 341bc9c56078586455583851873fbd264b8c2b6c7077c36b033c3d82a38d32d5 network default NAD default NAD key ] I0423 20:47:08.793328 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 341bc9c56078586, network: default, NAD default, SandboxID: "341bc9c56078586455583851873fbd264b8c2b6c7077c36b033c3d82a38d32d5", PCI device ID: , UID: "a595f314-f284-4e8d-b33b-e5c72733c1ad", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0423 20:47:08.844601 3825 cni.go:473] [openshift-monitoring/prometheus-k8s-0 341bc9c56078586455583851873fbd264b8c2b6c7077c36b033c3d82a38d32d5 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 341bc9c56078586455583851873fbd264b8c2b6c7077c36b033c3d82a38d32d5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:7d:9d:40:8e:7e\",\"name\":\"341bc9c56078586\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91c60dd9-e45d-4b75-bb2f-3d4595791217\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0423 20:47:42.031894 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0423 20:47:42.031925 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0423 20:47:52.907578 3825 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0423 20:48:13.239492 3825 cni.go:452] [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key ] I0423 20:48:13.317372 3825 cni.go:473] [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-86f657c86d-sbb9l 10004cf56ab1f9258a996dd71c6b516d08fe5efa90f0bcf70452707688344d3d network default NAD default NAD key default], result "{}", err I0423 20:48:13.370135 3825 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-sbb9l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:13.370175 3825 pods.go:176] Deleting pod: openshift-console/console-86f657c86d-sbb9l I0423 20:48:13.371435 3825 pods.go:236] Attempting to release IPs for pod: openshift-console/console-86f657c86d-sbb9l, ips: 10.130.0.21 I0423 20:48:13.371468 3825 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-sbb9l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:13.378916 3825 base_network_controller_pods.go:487] [default/openshift-console/console-66899c5bfc-mpwlr] creating logical port openshift-console_console-66899c5bfc-mpwlr for pod on switch ip-10-0-2-22.ec2.internal I0423 20:48:13.379037 3825 kube.go:256] Updating pod openshift-console/console-66899c5bfc-mpwlr I0423 20:48:13.394736 3825 pod.go:62] [openshift-console/console-66899c5bfc-mpwlr] pod update took 15.739027ms I0423 20:48:13.394767 3825 base_network_controller_pods.go:951] [default/openshift-console/console-66899c5bfc-mpwlr] addLogicalPort annotation time took 15.771016ms I0423 20:48:13.395944 3825 pods.go:271] [openshift-console/console-66899c5bfc-mpwlr] addLogicalPort took 17.045473ms, libovsdb time 849.175µs I0423 20:48:13.738374 3825 cni.go:452] [openshift-console/console-66899c5bfc-mpwlr bc8e37995955b5dd7c1ca775b3ac6e463ba0368df0ac3b39237fb1ced858fe74 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-66899c5bfc-mpwlr bc8e37995955b5dd7c1ca775b3ac6e463ba0368df0ac3b39237fb1ced858fe74 network default NAD default NAD key ] I0423 20:48:13.748501 3825 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-66899c5bfc-mpwlr, hostIfaceName: bc8e37995955b5d, network: default, NAD default, SandboxID: "bc8e37995955b5dd7c1ca775b3ac6e463ba0368df0ac3b39237fb1ced858fe74", PCI device ID: , UID: "d60a1566-65e5-4896-9a70-bb22104d9b98", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0423 20:48:13.798648 3825 cni.go:473] [openshift-console/console-66899c5bfc-mpwlr bc8e37995955b5dd7c1ca775b3ac6e463ba0368df0ac3b39237fb1ced858fe74 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-66899c5bfc-mpwlr bc8e37995955b5dd7c1ca775b3ac6e463ba0368df0ac3b39237fb1ced858fe74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:86:61:a2:ba:0e\",\"name\":\"bc8e37995955b5d\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9c6d48c8-fc43-4de5-b1cb-d8fa64f3fb87\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0423 20:48:14.135511 3825 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-sbb9l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:14.135540 3825 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-sbb9l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:14.143998 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86f657c86d-sbb9l I0423 20:48:14.144025 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86f657c86d-sbb9l W0423 20:48:24.177692 3825 util.go:959] Failed to build node endpoints for node ip-10-0-2-22.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0423 20:48:24.297316 3825 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-mhknr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:24.297850 3825 obj_retry.go:498] Detected object openshift-console/console-86f657c86d-mhknr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:48:25.255774 3825 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-mhknr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:25.255807 3825 obj_retry.go:492] Detected object openshift-console/console-86f657c86d-mhknr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:48:25.263065 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86f657c86d-mhknr I0423 20:48:25.263095 3825 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86f657c86d-mhknr I0423 20:48:35.668652 3825 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0423 20:49:00.581863 3825 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0423 20:49:10.581021 3825 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0423 20:49:39.705206 3825 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0423 20:49:48.711529 3825 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0423 20:50:04.531387 3825 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0423 20:50:04.531988 3825 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0423 20:50:05.448205 3825 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:50:05.448233 3825 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616290-pfkpr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0423 20:50:11.691813 3825 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0423 20:50:31.685956 3825 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0423 20:51:00.664743 3825 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0423 20:51:07.582738 3825 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=139 W0423 20:51:19.695405 3825 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0423 20:51:24.478979 3825 namespace.go:98] [tsf] adding namespace I0423 20:51:24.479946 3825 namespace.go:102] [tsf] adding namespace took 940.572µs I0423 20:51:24.520945 3825 namespace.go:142] [tsf] updating namespace I0423 20:51:24.530263 3825 namespace.go:142] [tsf] updating namespace I0423 20:51:30.744699 3825 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0423 20:51:30.744763 3825 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-v8sd6 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-v8sd6: no pod IPs found