+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-176.ec2.internal ++ [[ -n ip-10-0-1-176.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-176.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0420 08:56:15.557872349 - disable conntrack on geneve port' I0420 08:56:15.557872349 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0420 08:56:15.590521716 - starting ovnkube-node + echo 'I0420 08:56:15.590521716 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-176.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-176.ec2.internal --init-node ip-10-0-1-176.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-176.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0420 08:56:15.644070 3808 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0420 08:56:15.644305 3808 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-e4f814264e.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0420 08:56:15.645768 3808 kube.go:431] Waiting for certificate I0420 08:56:15.645795 3808 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0420 08:56:15.645846 3808 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0420 08:56:15.646692 3808 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0420 08:56:15.670209 3808 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 08:56:15.670227 3808 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 08:56:15.672288 3808 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 08:56:15.674329 3808 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-kjxg5" I0420 08:56:15.684747 3808 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-kjxg5" I0420 08:56:15.684823 3808 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0420 08:56:16.645969 3808 kube.go:438] Certificate found I0420 08:56:16.646586 3808 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0420 08:56:16.647523 3808 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0420 08:56:16.703409 3808 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-21 08:51:15 +0000 UTC" deadline="2026-04-21 06:17:11.087077345 +0000 UTC" I0420 08:56:16.703434 3808 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="21h20m54.383650547s" I0420 08:56:16.715916 3808 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0420 08:56:16.725847 3808 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0420 08:56:16.725917 3808 server.go:101] MetricServer registers OVS metrics I0420 08:56:16.726264 3808 server.go:105] MetricServer registers OVN DB metrics I0420 08:56:16.731548 3808 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0420 08:56:16.731605 3808 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0420 08:56:16.731844 3808 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0420 08:56:16.731967 3808 factory.go:561] Starting watch factory I0420 08:56:16.732123 3808 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732144 3808 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732560 3808 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732572 3808 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732581 3808 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732589 3808 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732777 3808 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732796 3808 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732832 3808 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.732857 3808 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.733143 3808 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.733203 3808 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.737745 3808 controller_manager.go:370] Starting the ovnkube controller I0420 08:56:16.737807 3808 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-176.ec2.internal I0420 08:56:16.737899 3808 controller_manager.go:395] NBDB zone sync took: 59.326µs I0420 08:56:16.737929 3808 factory.go:561] Starting watch factory I0420 08:56:16.746905 3808 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0420 08:56:16.746935 3808 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0420 08:56:16.750631 3808 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0420 08:56:16.750660 3808 server.go:109] MetricServer registers OVN Controller metrics I0420 08:56:16.754504 3808 server.go:113] MetricServer registers OVN Northd metrics I0420 08:56:16.755570 3808 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.757524 3808 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.757677 3808 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.758535 3808 metrics.go:519] OVN Metrics Server starts to run ... I0420 08:56:16.760845 3808 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.761152 3808 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.766785 3808 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0420 08:56:16.837313 3808 factory.go:1977] *v1.Node informer cache synced successfully I0420 08:56:16.837329 3808 factory.go:1977] *v1.Service informer cache synced successfully I0420 08:56:16.837333 3808 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0420 08:56:16.837337 3808 factory.go:1977] *v1.Pod informer cache synced successfully I0420 08:56:16.837340 3808 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0420 08:56:16.837343 3808 factory.go:1977] *v1.Namespace informer cache synced successfully I0420 08:56:16.837459 3808 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 08:56:16.837479 3808 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 08:56:16.837497 3808 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 08:56:16.837553 3808 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 08:56:16.841333 3808 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 08:56:16.841366 3808 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0420 08:56:16.843360 3808 factory.go:1977] *v1.Service informer cache synced successfully I0420 08:56:16.843373 3808 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0420 08:56:16.843377 3808 factory.go:1977] *v1.Pod informer cache synced successfully I0420 08:56:16.843380 3808 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0420 08:56:16.843383 3808 factory.go:1977] *v1.Namespace informer cache synced successfully I0420 08:56:16.843386 3808 factory.go:1977] *v1.Node informer cache synced successfully I0420 08:56:16.843434 3808 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0420 08:56:16.843446 3808 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0420 08:56:16.843501 3808 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.843518 3808 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.846905 3808 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.848895 3808 factory.go:1977] *v1.EgressIP informer cache synced successfully I0420 08:56:16.848986 3808 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0420 08:56:16.849000 3808 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0420 08:56:16.849038 3808 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.849057 3808 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.852234 3808 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.854200 3808 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0420 08:56:16.854204 3808 factory.go:1977] *v1.EgressIP informer cache synced successfully I0420 08:56:16.854287 3808 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.854302 3808 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.857497 3808 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.859486 3808 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0420 08:56:16.859557 3808 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0420 08:56:16.859617 3808 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.859629 3808 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.862867 3808 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.864887 3808 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0420 08:56:16.864898 3808 factory.go:1977] *v1.EgressService informer cache synced successfully I0420 08:56:16.864966 3808 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.864988 3808 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.868221 3808 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.870194 3808 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0420 08:56:16.870277 3808 factory.go:1977] *v1.EgressService informer cache synced successfully I0420 08:56:16.870370 3808 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0420 08:56:16.870391 3808 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0420 08:56:16.873935 3808 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0420 08:56:16.875890 3808 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0420 08:56:16.875962 3808 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.875975 3808 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.876008 3808 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.876027 3808 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.876071 3808 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0420 08:56:16.879396 3808 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.879906 3808 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0420 08:56:16.881938 3808 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0420 08:56:16.881948 3808 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0420 08:56:16.881955 3808 factory.go:680] Watch Factory start up complete, took: 149.993661ms I0420 08:56:16.882017 3808 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0420 08:56:16.882039 3808 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0420 08:56:16.887281 3808 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0420 08:56:16.887291 3808 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0420 08:56:16.887298 3808 factory.go:680] Watch Factory start up complete, took: 149.369767ms I0420 08:56:16.887309 3808 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-176.ec2.internal" zone I0420 08:56:16.934492 3808 default_node_network_controller.go:710] Initializing the default node network controller I0420 08:56:17.022664 3808 udn_isolation.go:101] Starting UDN host isolation manager I0420 08:56:17.028840 3808 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0420 08:56:17.085323 3808 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0420 08:56:17.085383 3808 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0420 08:56:17.085396 3808 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0420 08:56:17.120228 3808 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0420 08:56:17.120282 3808 default_node_network_controller.go:809] Node ip-10-0-1-176.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0420 08:56:17.126046 3808 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.176"] k8s.ovn.org/zone-name:ip-10-0-1-176.ec2.internal] on node ip-10-0-1-176.ec2.internal I0420 08:56:17.146986 3808 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0420 08:56:17.153053 3808 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0420 08:56:17.153175 3808 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0420 08:56:17.153312 3808 gateway_init.go:263] Preparing Gateway I0420 08:56:17.153325 3808 gateway_shared_intf.go:1687] Creating new gateway I0420 08:56:17.258944 3808 iptables.go:108] Creating table: filter chain: FORWARD I0420 08:56:17.264586 3808 iptables.go:108] Creating table: filter chain: OUTPUT I0420 08:56:17.270428 3808 gateway_shared_intf.go:1816] Gateway Creation Complete I0420 08:56:17.270442 3808 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-176.ec2.internal","mac-address":"0a:ff:f1:87:11:d5","ip-addresses":["10.0.1.176/24"],"ip-address":"10.0.1.176/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:4c542c59-55ec-4efc-958a-7f493c16728a k8s.ovn.org/node-encap-ips:["10.0.1.176"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.176/24"} k8s.ovn.org/zone-name:ip-10-0-1-176.ec2.internal] on node ip-10-0-1-176.ec2.internal I0420 08:56:17.289716 3808 default_node_network_controller.go:519] Node connection status = connected I0420 08:56:17.388221 3808 controller_manager.go:429] Waiting for node in zone sync took: 500.901453ms I0420 08:56:17.394247 3808 controller_manager.go:611] Switching to transit router for layer2 networks I0420 08:56:17.394261 3808 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-176.ec2.internal I0420 08:56:17.409929 3808 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0420 08:56:17.415991 3808 controller_manager.go:290] SCTP support detected in OVN I0420 08:56:17.416625 3808 services_controller.go:69] Creating services controller for network=default I0420 08:56:17.416679 3808 udn_enabled_svc.go:80] Waiting for service informer to sync I0420 08:56:17.416745 3808 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0420 08:56:17.416774 3808 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0420 08:56:17.416785 3808 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0420 08:56:17.416794 3808 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0420 08:56:17.416987 3808 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0420 08:56:17.417041 3808 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0420 08:56:17.417073 3808 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 51.852µs I0420 08:56:17.417087 3808 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 34.997µs I0420 08:56:17.417115 3808 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 55.9µs I0420 08:56:17.417124 3808 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0420 08:56:17.417139 3808 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0420 08:56:17.417154 3808 default_network_controller.go:325] Starting the default network controller I0420 08:56:17.417174 3808 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 17.277µs I0420 08:56:17.417204 3808 udn_enabled_svc.go:95] Performing full resync I0420 08:56:17.417536 3808 udn_enabled_svc.go:99] Waiting for handler to sync I0420 08:56:17.417546 3808 udn_enabled_svc.go:104] Starting worker I0420 08:56:17.426359 3808 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0420 08:56:17.426416 3808 repair.go:33] Syncing exgw routes took 44.042µs I0420 08:56:17.426440 3808 default_network_controller.go:414] Starting all the Watchers... I0420 08:56:17.426580 3808 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0420 08:56:17.426614 3808 namespace.go:98] [openshift-dns] adding namespace I0420 08:56:17.426622 3808 namespace.go:98] [openshift-kube-apiserver] adding namespace I0420 08:56:17.426623 3808 namespace.go:98] [openshift-authentication-operator] adding namespace I0420 08:56:17.426631 3808 namespace.go:98] [openshift-console-operator] adding namespace I0420 08:56:17.426634 3808 namespace.go:98] [openshift-apiserver-operator] adding namespace I0420 08:56:17.426636 3808 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0420 08:56:17.426643 3808 namespace.go:98] [openshift-network-operator] adding namespace I0420 08:56:17.426639 3808 namespace.go:98] [openshift-customer-monitoring] adding namespace I0420 08:56:17.426646 3808 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0420 08:56:17.426655 3808 namespace.go:98] [openshift-operators] adding namespace I0420 08:56:17.426653 3808 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0420 08:56:17.426659 3808 namespace.go:98] [openshift-insights] adding namespace I0420 08:56:17.426663 3808 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0420 08:56:17.426688 3808 namespace.go:98] [openshift-network-diagnostics] adding namespace I0420 08:56:17.427172 3808 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 578.004µs I0420 08:56:17.427189 3808 namespace.go:98] [openshift-ingress-operator] adding namespace I0420 08:56:17.427712 3808 namespace.go:102] [openshift-ingress-operator] adding namespace took 518.369µs I0420 08:56:17.427723 3808 namespace.go:98] [openshift-kube-scheduler] adding namespace I0420 08:56:17.428353 3808 namespace.go:102] [openshift-kube-scheduler] adding namespace took 625.17µs I0420 08:56:17.428366 3808 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0420 08:56:17.428955 3808 namespace.go:102] [openshift-dns] adding namespace took 2.327861ms I0420 08:56:17.428977 3808 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0420 08:56:17.429503 3808 namespace.go:102] [openshift-kube-apiserver] adding namespace took 2.872286ms I0420 08:56:17.429517 3808 namespace.go:98] [openshift-network-node-identity] adding namespace I0420 08:56:17.430130 3808 namespace.go:102] [openshift-authentication-operator] adding namespace took 3.493289ms I0420 08:56:17.430153 3808 namespace.go:98] [openshift-config] adding namespace I0420 08:56:17.430696 3808 namespace.go:102] [openshift-console-operator] adding namespace took 4.054981ms I0420 08:56:17.430717 3808 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0420 08:56:17.431262 3808 namespace.go:102] [openshift-apiserver-operator] adding namespace took 4.620729ms I0420 08:56:17.431280 3808 namespace.go:98] [openshift-authentication] adding namespace I0420 08:56:17.431798 3808 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.148447ms I0420 08:56:17.431815 3808 namespace.go:98] [openshift-host-network] adding namespace I0420 08:56:17.432453 3808 namespace.go:102] [openshift-network-operator] adding namespace took 5.803729ms I0420 08:56:17.432464 3808 namespace.go:98] [openshift-logging] adding namespace I0420 08:56:17.433052 3808 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.402914ms I0420 08:56:17.433062 3808 namespace.go:98] [openshift-machine-api] adding namespace I0420 08:56:17.433646 3808 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.991359ms I0420 08:56:17.433665 3808 namespace.go:98] [openshift-backplane-mobb] adding namespace I0420 08:56:17.434062 3808 namespace.go:102] [openshift-operators] adding namespace took 7.400947ms I0420 08:56:17.434079 3808 namespace.go:98] [openshift-infra] adding namespace I0420 08:56:17.434678 3808 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 8.01725ms I0420 08:56:17.434690 3808 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0420 08:56:17.435254 3808 namespace.go:102] [openshift-insights] adding namespace took 8.588364ms I0420 08:56:17.435269 3808 namespace.go:98] [openshift-ingress-canary] adding namespace W0420 08:56:17.435808 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-5qmmr: pod openshift-network-diagnostics/network-check-target-5qmmr: no pod IPs found W0420 08:56:17.435822 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rbwxv: pod openshift-network-diagnostics/network-check-target-rbwxv: no pod IPs found I0420 08:56:17.435848 3808 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 9.175994ms I0420 08:56:17.435857 3808 namespace.go:98] [openshift-backplane] adding namespace I0420 08:56:17.436453 3808 namespace.go:102] [openshift-network-diagnostics] adding namespace took 9.753536ms I0420 08:56:17.436467 3808 namespace.go:98] [kube-node-lease] adding namespace I0420 08:56:17.436919 3808 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 8.545893ms I0420 08:56:17.436941 3808 namespace.go:98] [openshift-backplane-tam] adding namespace I0420 08:56:17.437467 3808 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.481482ms I0420 08:56:17.437488 3808 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0420 08:56:17.437946 3808 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.421723ms I0420 08:56:17.437964 3808 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0420 08:56:17.438475 3808 namespace.go:102] [openshift-config] adding namespace took 8.315421ms I0420 08:56:17.438490 3808 namespace.go:98] [kube-system] adding namespace I0420 08:56:17.438878 3808 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.155647ms I0420 08:56:17.438890 3808 namespace.go:98] [openshift-operators-redhat] adding namespace I0420 08:56:17.439490 3808 namespace.go:102] [openshift-authentication] adding namespace took 8.189756ms I0420 08:56:17.439501 3808 namespace.go:98] [openshift-backplane-srep] adding namespace I0420 08:56:17.439920 3808 namespace.go:102] [openshift-host-network] adding namespace took 8.098436ms I0420 08:56:17.439931 3808 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0420 08:56:17.440481 3808 namespace.go:102] [openshift-logging] adding namespace took 8.012478ms I0420 08:56:17.440493 3808 namespace.go:98] [openshift-multus] adding namespace I0420 08:56:17.440981 3808 namespace.go:102] [openshift-machine-api] adding namespace took 7.912008ms I0420 08:56:17.440998 3808 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0420 08:56:17.441518 3808 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.846525ms I0420 08:56:17.441531 3808 namespace.go:98] [openshift-console-user-settings] adding namespace I0420 08:56:17.441887 3808 namespace.go:102] [openshift-infra] adding namespace took 7.801033ms I0420 08:56:17.441903 3808 namespace.go:98] [openshift-backplane-cse] adding namespace I0420 08:56:17.442456 3808 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.760776ms I0420 08:56:17.442469 3808 namespace.go:98] [openshift-must-gather-operator] adding namespace I0420 08:56:17.442963 3808 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.687393ms I0420 08:56:17.442981 3808 namespace.go:98] [openshift-cluster-version] adding namespace I0420 08:56:17.443428 3808 namespace.go:102] [openshift-backplane] adding namespace took 7.564236ms I0420 08:56:17.443444 3808 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0420 08:56:17.443873 3808 namespace.go:102] [kube-node-lease] adding namespace took 7.397847ms I0420 08:56:17.443889 3808 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0420 08:56:17.444429 3808 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.48041ms I0420 08:56:17.444450 3808 namespace.go:98] [openshift-console] adding namespace I0420 08:56:17.444919 3808 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.424247ms I0420 08:56:17.444933 3808 namespace.go:98] [openshift-route-controller-manager] adding namespace I0420 08:56:17.445465 3808 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.495539ms I0420 08:56:17.445478 3808 namespace.go:98] [openshift-marketplace] adding namespace I0420 08:56:17.446008 3808 namespace.go:102] [kube-system] adding namespace took 7.509579ms I0420 08:56:17.446027 3808 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0420 08:56:17.446537 3808 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.642713ms I0420 08:56:17.446550 3808 namespace.go:98] [openshift-etcd] adding namespace I0420 08:56:17.447087 3808 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.580614ms I0420 08:56:17.447114 3808 namespace.go:98] [openshift-network-console] adding namespace W0420 08:56:17.447610 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-qgnzf: pod openshift-multus/network-metrics-daemon-qgnzf: no pod IPs found W0420 08:56:17.447621 3808 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-ln6xc: pod openshift-multus/network-metrics-daemon-ln6xc: no pod IPs found I0420 08:56:17.447665 3808 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.728043ms I0420 08:56:17.447683 3808 namespace.go:98] [openshift-node] adding namespace I0420 08:56:17.448160 3808 namespace.go:102] [openshift-multus] adding namespace took 7.661211ms I0420 08:56:17.448174 3808 namespace.go:98] [openshift-image-registry] adding namespace I0420 08:56:17.448781 3808 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.777252ms I0420 08:56:17.448798 3808 namespace.go:98] [kube-public] adding namespace I0420 08:56:17.449347 3808 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.809644ms I0420 08:56:17.449365 3808 namespace.go:98] [openshift-monitoring] adding namespace I0420 08:56:17.449766 3808 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.857911ms I0420 08:56:17.449783 3808 namespace.go:98] [dedicated-admin] adding namespace I0420 08:56:17.450433 3808 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.958705ms I0420 08:56:17.450445 3808 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0420 08:56:17.450817 3808 namespace.go:102] [openshift-cluster-version] adding namespace took 7.828489ms I0420 08:56:17.450833 3808 namespace.go:98] [openshift-apiserver] adding namespace I0420 08:56:17.451408 3808 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.959234ms I0420 08:56:17.451422 3808 namespace.go:98] [default] adding namespace I0420 08:56:17.451955 3808 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.058776ms I0420 08:56:17.451972 3808 namespace.go:98] [openshift-machine-config-operator] adding namespace I0420 08:56:17.452503 3808 namespace.go:102] [openshift-console] adding namespace took 8.045462ms I0420 08:56:17.452934 3808 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.995262ms I0420 08:56:17.452948 3808 namespace.go:98] [openshift-backplane-cee] adding namespace I0420 08:56:17.453571 3808 namespace.go:102] [openshift-marketplace] adding namespace took 8.085722ms I0420 08:56:17.453589 3808 namespace.go:98] [openshift-package-operator] adding namespace I0420 08:56:17.454002 3808 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.968351ms I0420 08:56:17.454017 3808 namespace.go:98] [openshift-backplane-csm] adding namespace I0420 08:56:17.455023 3808 namespace.go:102] [openshift-etcd] adding namespace took 8.467464ms I0420 08:56:17.455036 3808 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0420 08:56:17.455406 3808 namespace.go:102] [openshift-network-console] adding namespace took 8.286081ms I0420 08:56:17.455417 3808 namespace.go:98] [openshift-service-ca-operator] adding namespace I0420 08:56:17.455937 3808 namespace.go:102] [openshift-node] adding namespace took 8.245739ms I0420 08:56:17.455952 3808 namespace.go:98] [open-cluster-management-2ppph9ruf6k5ml3crseffj0vunji1dnc] adding namespace I0420 08:56:17.456457 3808 namespace.go:102] [openshift-image-registry] adding namespace took 8.275428ms I0420 08:56:17.456473 3808 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0420 08:56:17.456924 3808 namespace.go:102] [kube-public] adding namespace took 8.118444ms I0420 08:56:17.456941 3808 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0420 08:56:17.457401 3808 namespace.go:102] [openshift-monitoring] adding namespace took 8.029352ms I0420 08:56:17.457414 3808 namespace.go:98] [openshift-dns-operator] adding namespace I0420 08:56:17.457915 3808 namespace.go:102] [dedicated-admin] adding namespace took 8.127259ms I0420 08:56:17.457928 3808 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0420 08:56:17.458332 3808 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.879747ms I0420 08:56:17.458349 3808 namespace.go:98] [openshift-controller-manager] adding namespace I0420 08:56:17.458859 3808 namespace.go:102] [openshift-apiserver] adding namespace took 8.020635ms I0420 08:56:17.459278 3808 namespace.go:102] [default] adding namespace took 7.851246ms I0420 08:56:17.459295 3808 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0420 08:56:17.459755 3808 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.773916ms I0420 08:56:17.459776 3808 namespace.go:98] [openshift-config-operator] adding namespace I0420 08:56:17.460226 3808 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.272382ms I0420 08:56:17.460239 3808 namespace.go:98] [openshift] adding namespace I0420 08:56:17.460750 3808 namespace.go:102] [openshift-package-operator] adding namespace took 7.155044ms I0420 08:56:17.461121 3808 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.099072ms I0420 08:56:17.461713 3808 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.669731ms I0420 08:56:17.462215 3808 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.791876ms I0420 08:56:17.462231 3808 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0420 08:56:17.462699 3808 namespace.go:102] [open-cluster-management-2ppph9ruf6k5ml3crseffj0vunji1dnc] adding namespace took 6.742314ms I0420 08:56:17.462713 3808 namespace.go:98] [openshift-config-managed] adding namespace I0420 08:56:17.463117 3808 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.638225ms I0420 08:56:17.463576 3808 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.629282ms I0420 08:56:17.464117 3808 namespace.go:102] [openshift-dns-operator] adding namespace took 6.698628ms I0420 08:56:17.464582 3808 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.649117ms I0420 08:56:17.465012 3808 namespace.go:102] [openshift-controller-manager] adding namespace took 6.658363ms I0420 08:56:17.465543 3808 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.242668ms I0420 08:56:17.465555 3808 namespace.go:98] [openshift-ingress] adding namespace I0420 08:56:17.466045 3808 namespace.go:102] [openshift-config-operator] adding namespace took 6.2632ms I0420 08:56:17.466578 3808 namespace.go:102] [openshift] adding namespace took 6.333984ms I0420 08:56:17.467121 3808 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 4.883926ms I0420 08:56:17.467558 3808 namespace.go:102] [openshift-config-managed] adding namespace took 4.84065ms I0420 08:56:17.468088 3808 namespace.go:102] [openshift-ingress] adding namespace took 2.527349ms W0420 08:56:17.468174 3808 master.go:471] Unable to parse remote node ip-10-0-0-221.ec2.internal chassis-id annotation. Chassis may be removed during sync I0420 08:56:17.468257 3808 master.go:548] Adding or Updating local node "ip-10-0-1-176.ec2.internal" for network "default" I0420 08:56:17.468300 3808 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-221.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-221.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-221.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-221.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-221.ec2.internal W0420 08:56:17.475482 3808 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-qgnzf: pod openshift-multus/network-metrics-daemon-qgnzf: no pod IPs found W0420 08:56:17.475499 3808 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-rbwxv: pod openshift-network-diagnostics/network-check-target-rbwxv: no pod IPs found I0420 08:56:17.476356 3808 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-176.ec2.internal for the network default I0420 08:56:17.476676 3808 zone_ic_handler.go:195] Time taken to create transit switch: 307.413µs I0420 08:56:17.477406 3808 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0420 08:56:17.477426 3808 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0420 08:56:17.477436 3808 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0420 08:56:17.477459 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0420 08:56:17.477481 3808 node_tracker.go:235] Node ip-10-0-0-221.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-221.ec2.internal" I0420 08:56:17.477491 3808 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:17.477502 3808 services_controller.go:574] Full service sync requested for network=default I0420 08:56:17.477510 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0420 08:56:17.477537 3808 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:17.477543 3808 services_controller.go:574] Full service sync requested for network=default I0420 08:56:17.577601 3808 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0420 08:56:17.577617 3808 services_controller.go:208] Setting up event handlers for services for network=default I0420 08:56:17.577687 3808 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0420 08:56:17.577727 3808 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0420 08:56:17.577741 3808 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0420 08:56:17.677921 3808 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0420 08:56:17.677940 3808 repair.go:60] Starting repairing loop for services I0420 08:56:17.678021 3808 repair.go:131] Deleted 0 stale service LBs I0420 08:56:17.678044 3808 repair.go:137] Deleted 0 stale Chassis Template Vars I0420 08:56:17.678066 3808 repair.go:62] Finished repairing loop for services: 130.313µs I0420 08:56:17.678122 3808 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0420 08:56:17.678136 3808 services_controller.go:254] Starting workers for network=default I0420 08:56:17.678407 3808 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-ln6xc, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-ln6xc: suppressed error logged: pod openshift-multus/network-metrics-daemon-ln6xc: no pod IPs found I0420 08:56:17.678459 3808 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-qgnzf] creating logical port openshift-multus_network-metrics-daemon-qgnzf for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:17.678504 3808 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-rbwxv] creating logical port openshift-network-diagnostics_network-check-target-rbwxv for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:17.678597 3808 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-qgnzf I0420 08:56:17.678600 3808 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-rbwxv I0420 08:56:17.678931 3808 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-5qmmr, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5qmmr: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5qmmr: no pod IPs found I0420 08:56:17.696312 3808 pod.go:62] [openshift-network-diagnostics/network-check-target-rbwxv] pod update took 17.742686ms I0420 08:56:17.696335 3808 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-rbwxv] addLogicalPort annotation time took 17.768633ms I0420 08:56:17.696342 3808 pod.go:62] [openshift-multus/network-metrics-daemon-qgnzf] pod update took 17.820009ms I0420 08:56:17.696362 3808 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-qgnzf] addLogicalPort annotation time took 17.839818ms I0420 08:56:17.697250 3808 pods.go:271] [openshift-network-diagnostics/network-check-target-rbwxv] addLogicalPort took 18.753675ms, libovsdb time 709.741µs I0420 08:56:17.697510 3808 pods.go:271] [openshift-multus/network-metrics-daemon-qgnzf] addLogicalPort took 19.057873ms, libovsdb time 892.999µs I0420 08:56:17.697908 3808 admin_network_policy_controller.go:227] Starting controller default-network-controller I0420 08:56:17.697929 3808 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0420 08:56:17.697942 3808 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0420 08:56:17.697946 3808 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0420 08:56:17.698133 3808 repair.go:29] Repairing admin network policies took 181.314µs I0420 08:56:17.698245 3808 repair.go:92] Repairing baseline admin network policies took 100.946µs I0420 08:56:17.698549 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0420 08:56:17.698567 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0420 08:56:17.700075 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0420 08:56:17.700095 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0420 08:56:17.700123 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0420 08:56:17.702039 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0420 08:56:17.702055 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0420 08:56:17.702063 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0420 08:56:17.702547 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0420 08:56:17.702567 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0420 08:56:17.702576 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0420 08:56:17.703079 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0420 08:56:17.703110 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0420 08:56:17.703120 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0420 08:56:17.704506 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0420 08:56:17.704522 3808 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0420 08:56:17.704531 3808 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0420 08:56:17.706407 3808 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0420 08:56:17.706424 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0420 08:56:17.706435 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0420 08:56:17.708447 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0420 08:56:17.708468 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0420 08:56:17.708478 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0420 08:56:17.709326 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0420 08:56:17.709340 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0420 08:56:17.709350 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0420 08:56:17.710260 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0420 08:56:17.710282 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0420 08:56:17.710291 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0420 08:56:17.711742 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0420 08:56:17.711762 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0420 08:56:17.711772 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0420 08:56:17.712423 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0420 08:56:17.712441 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0420 08:56:17.712831 3808 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0420 08:56:17.712844 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0420 08:56:17.713507 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0420 08:56:17.713527 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0420 08:56:17.713534 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0420 08:56:17.714006 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0420 08:56:17.714024 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0420 08:56:17.714042 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0420 08:56:17.714717 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0420 08:56:17.714736 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0420 08:56:17.714745 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0420 08:56:17.715204 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0420 08:56:17.715221 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0420 08:56:17.715228 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0420 08:56:17.716866 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0420 08:56:17.716883 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0420 08:56:17.716891 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0420 08:56:17.717823 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0420 08:56:17.717841 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0420 08:56:17.717847 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0420 08:56:17.718111 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0420 08:56:17.718128 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0420 08:56:17.718137 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0420 08:56:17.718607 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0420 08:56:17.718625 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0420 08:56:17.718633 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0420 08:56:17.719081 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0420 08:56:17.719112 3808 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0420 08:56:17.719121 3808 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0420 08:56:17.719594 3808 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0420 08:56:17.719610 3808 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0420 08:56:17.719619 3808 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0420 08:56:17.720534 3808 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0420 08:56:17.720552 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0420 08:56:17.720560 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0420 08:56:17.720812 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0420 08:56:17.720830 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0420 08:56:17.720838 3808 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0420 08:56:17.721296 3808 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0420 08:56:17.721313 3808 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0420 08:56:17.721319 3808 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0420 08:56:17.721575 3808 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0420 08:56:17.721590 3808 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0420 08:56:17.721596 3808 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0420 08:56:17.721831 3808 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0420 08:56:17.721844 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0420 08:56:17.721862 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0420 08:56:17.722858 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0420 08:56:17.722875 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0420 08:56:17.722884 3808 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0420 08:56:17.723979 3808 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0420 08:56:17.723995 3808 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0420 08:56:17.724005 3808 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0420 08:56:17.725482 3808 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0420 08:56:17.729951 3808 egressfirewall.go:374] Starting EgressFirewall controller I0420 08:56:17.729987 3808 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0420 08:56:17.729989 3808 controller.go:133] Adding controller egress-firewall-controller event handlers I0420 08:56:17.730013 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0420 08:56:17.730028 3808 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0420 08:56:17.730029 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0420 08:56:17.730038 3808 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0420 08:56:17.730294 3808 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0420 08:56:17.730308 3808 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0420 08:56:17.730313 3808 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0420 08:56:17.730322 3808 egressqos.go:193] Setting up event handlers for EgressQoS I0420 08:56:17.730327 3808 egressfirewall.go:723] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0420 08:56:17.730334 3808 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0420 08:56:17.730405 3808 egressqos.go:245] Starting EgressQoS Controller I0420 08:56:17.730415 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0420 08:56:17.730421 3808 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0420 08:56:17.730426 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0420 08:56:17.730430 3808 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0420 08:56:17.730435 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0420 08:56:17.730439 3808 shared_informer.go:356] "Caches are synced" controller="egressqos" I0420 08:56:17.730443 3808 egressqos.go:259] Repairing EgressQoSes I0420 08:56:17.730447 3808 egressqos.go:399] Starting repairing loop for egressqos I0420 08:56:17.730508 3808 egressqos.go:401] Finished repairing loop for egressqos: 60.987µs I0420 08:56:17.730524 3808 egressservice_zone.go:132] Setting up event handlers for Egress Services I0420 08:56:17.730554 3808 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-176.ec2.internal I0420 08:56:17.730594 3808 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-176.ec2.internal : 41.014µs I0420 08:56:17.730655 3808 egressservice_zone.go:210] Starting Egress Services Controller I0420 08:56:17.730668 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0420 08:56:17.730676 3808 shared_informer.go:356] "Caches are synced" controller="egressservices" I0420 08:56:17.730684 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0420 08:56:17.730691 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0420 08:56:17.730699 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0420 08:56:17.730706 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0420 08:56:17.730713 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0420 08:56:17.730720 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0420 08:56:17.730726 3808 egressservice_zone.go:228] Repairing Egress Services I0420 08:56:17.730996 3808 master_controller.go:88] Starting Admin Policy Based Route Controller I0420 08:56:17.731005 3808 external_controller.go:278] Starting Admin Policy Based Route Controller I0420 08:56:17.731020 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0420 08:56:17.731036 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 14.722µs I0420 08:56:17.731043 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0420 08:56:17.731047 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 4.616µs I0420 08:56:17.731140 3808 default_network_controller.go:585] Completing all the Watchers took 304.688138ms I0420 08:56:17.731158 3808 default_network_controller.go:589] Starting unidling controllers I0420 08:56:17.731171 3808 unidle.go:46] Registering OVN SB ControllerEvent handler I0420 08:56:17.731179 3808 unidle.go:63] Populating Initial ContollerEvent events I0420 08:56:17.731192 3808 unidle.go:79] Setting up event handlers for services I0420 08:56:17.731255 3808 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0420 08:56:17.790355 3808 default_node_network_controller.go:519] Node connection status = connected I0420 08:56:18.291232 3808 default_node_network_controller.go:519] Node connection status = connected I0420 08:56:18.809931 3808 default_node_network_controller.go:519] Node connection status = connected I0420 08:56:18.877023 3808 bridgeconfig.go:537] Gateway is ready I0420 08:56:18.877044 3808 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0420 08:56:18.888662 3808 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.176. Updating OVN encap IP. I0420 08:56:18.891862 3808 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0420 08:56:18.893849 3808 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.176 - it is already configured I0420 08:56:18.893863 3808 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.176/24:{} 172.20.0.1/32:{}]. Updating annotations. I0420 08:56:18.894080 3808 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.176/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-176.ec2.internal","mac-address":"0a:ff:f1:87:11:d5","ip-addresses":["10.0.1.176/24"],"ip-address":"10.0.1.176/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:4c542c59-55ec-4efc-958a-7f493c16728a k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.176/24"}] on node ip-10-0-1-176.ec2.internal I0420 08:56:18.912616 3808 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.176 I0420 08:56:18.912728 3808 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-176.ec2.internal I0420 08:56:18.912761 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0420 08:56:18.912801 3808 master.go:548] Adding or Updating local node "ip-10-0-1-176.ec2.internal" for network "default" I0420 08:56:18.912818 3808 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:18.912833 3808 services_controller.go:574] Full service sync requested for network=default I0420 08:56:18.912840 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0420 08:56:18.912852 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 17.725µs I0420 08:56:18.912868 3808 egressfirewall.go:723] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0420 08:56:18.917013 3808 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-176.ec2.internal. Updating no re-route policies I0420 08:56:18.917107 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-176.ec2.internal I0420 08:56:18.917117 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-76x95 I0420 08:56:18.917123 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-176.ec2.internal I0420 08:56:18.917131 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-76x95 I0420 08:56:18.917133 3808 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-176.ec2.internal I0420 08:56:18.917140 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rbwxv I0420 08:56:18.917121 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-fh5sv I0420 08:56:18.917148 3808 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-176.ec2.internal I0420 08:56:18.917152 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-fh5sv I0420 08:56:18.917153 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rbwxv I0420 08:56:18.917152 3808 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-bs6qh I0420 08:56:18.917154 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-s4qsb I0420 08:56:18.917165 3808 obj_retry.go:398] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-bs6qh I0420 08:56:18.917167 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-fh5sv after 0 failed attempt(s) I0420 08:56:18.917169 3808 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-176.ec2.internal after 0 failed attempt(s) I0420 08:56:18.917129 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-9w6rr I0420 08:56:18.917179 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-9w6rr I0420 08:56:18.917169 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-s4qsb I0420 08:56:18.917183 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-64r6f I0420 08:56:18.917183 3808 obj_retry.go:418] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-bs6qh after 0 failed attempt(s) I0420 08:56:18.917182 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-vhsrg I0420 08:56:18.917188 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-64r6f I0420 08:56:18.917197 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-s4qsb after 0 failed attempt(s) I0420 08:56:18.917199 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-9w6rr after 0 failed attempt(s) I0420 08:56:18.917201 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-ln6xc I0420 08:56:18.917142 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-xt4p7 I0420 08:56:18.917211 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-ln6xc I0420 08:56:18.917216 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-xt4p7 I0420 08:56:18.917227 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-xt4p7 after 0 failed attempt(s) I0420 08:56:18.917184 3808 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-rbwxv] creating logical port openshift-network-diagnostics_network-check-target-rbwxv for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:18.917248 3808 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-ln6xc, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-ln6xc: suppressed error logged: pod openshift-multus/network-metrics-daemon-ln6xc: no pod IPs found I0420 08:56:18.917198 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-vhsrg I0420 08:56:18.917271 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-vhsrg after 0 failed attempt(s) I0420 08:56:18.917167 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-cvsxb I0420 08:56:18.917281 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-cvsxb I0420 08:56:18.917291 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-cvsxb after 0 failed attempt(s) I0420 08:56:18.917144 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-176.ec2.internal after 0 failed attempt(s) I0420 08:56:18.917183 3808 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-qgnzf I0420 08:56:18.917302 3808 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-qgnzf I0420 08:56:18.917168 3808 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-l4bdq I0420 08:56:18.917321 3808 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-l4bdq I0420 08:56:18.917326 3808 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-qgnzf] creating logical port openshift-multus_network-metrics-daemon-qgnzf for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:18.917335 3808 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-l4bdq after 0 failed attempt(s) I0420 08:56:18.917156 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-76x95 after 0 failed attempt(s) I0420 08:56:18.917202 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-64r6f after 0 failed attempt(s) I0420 08:56:18.917853 3808 pods.go:271] [openshift-network-diagnostics/network-check-target-rbwxv] addLogicalPort took 679.236µs, libovsdb time 222.607µs I0420 08:56:18.917863 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-rbwxv after 0 failed attempt(s) I0420 08:56:18.918135 3808 pods.go:271] [openshift-multus/network-metrics-daemon-qgnzf] addLogicalPort took 817.762µs, libovsdb time 481.786µs I0420 08:56:18.918143 3808 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-qgnzf after 0 failed attempt(s) I0420 08:56:18.929418 3808 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0420 08:56:18.940539 3808 iptables.go:108] Creating table: mangle chain: OUTPUT I0420 08:56:18.943659 3808 iptables.go:108] Creating table: nat chain: OUTPUT I0420 08:56:18.993276 3808 iptables.go:108] Creating table: nat chain: PREROUTING I0420 08:56:19.057195 3808 iptables.go:108] Creating table: filter chain: FORWARD I0420 08:56:19.072821 3808 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0420 08:56:19.072904 3808 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0420 08:56:19.072953 3808 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0420 08:56:19.073018 3808 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0420 08:56:19.073065 3808 gateway_init.go:325] Gateway and management port readiness took 1.786782198s I0420 08:56:19.073090 3808 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0420 08:56:19.073134 3808 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0420 08:56:19.073145 3808 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0420 08:56:19.073154 3808 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0420 08:56:19.073217 3808 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0420 08:56:19.073290 3808 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.464µs I0420 08:56:19.073483 3808 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0420 08:56:19.073504 3808 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 13.248µs I0420 08:56:19.073512 3808 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 29.479µs I0420 08:56:19.073517 3808 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0420 08:56:19.073525 3808 nad_controller.go:379] [node-nad-controller NAD controller]: started I0420 08:56:19.073530 3808 default_node_network_controller.go:899] Starting the default node network controller I0420 08:56:19.073534 3808 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0420 08:56:19.073607 3808 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 11.094µs I0420 08:56:19.073654 3808 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0420 08:56:19.073662 3808 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0420 08:56:19.073683 3808 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c20360 0xc0002ed8f0 0xc0002ed960 0xc0002ed9d0 0xc0002eda40 0xc0002edb20 0xc0002edb90 0xc000c52780 0xc0002edc00 0xc0002edc70 0xc000c52820 0xc006133340 0xc0002edce0 map[0x2ff9080:0xc001b28960 0x2ff95c0:0xc001b28f60 0x2ffa2e0:0xc001b288a0 0x2ffa580:0xc001b28900 0x2ffad60:0xc001b289c0 0x2ffb000:0xc001b28a20 0x2ffb2a0:0xc001b28a80 0x2ffba80:0xc001b28fc0 0x2ffbd20:0xc001b29020 0x3026980:0xc001b28240 0x3026d20:0xc001b282a0 0x3027800:0xc000c66360 0x3029160:0xc001b28000 0x30298a0:0xc001b283c0 0x302a720:0xc001b281e0] 0xc000126620 0} I0420 08:56:19.073730 3808 gateway.go:151] Starting gateway service sync I0420 08:56:19.074094 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0420 08:56:19.076122 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0420 08:56:19.095543 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0420 08:56:19.096582 3808 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0420 08:56:19.097617 3808 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0420 08:56:19.130162 3808 gateway.go:168] Gateway service sync done. Time taken: 56.421167ms I0420 08:56:19.130560 3808 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31233 I0420 08:56:19.130949 3808 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c20360 0xc0002ed8f0 0xc0002ed960 0xc0002ed9d0 0xc0002eda40 0xc0002edb20 0xc0002edb90 0xc000c52780 0xc0002edc00 0xc0002edc70 0xc000c52820 0xc006133340 0xc0002edce0 map[0x2ff9080:0xc001b28960 0x2ff95c0:0xc001b28f60 0x2ffa2e0:0xc001b288a0 0x2ffa580:0xc001b28900 0x2ffad60:0xc001b289c0 0x2ffb000:0xc001b28a20 0x2ffb2a0:0xc001b28a80 0x2ffba80:0xc001b28fc0 0x2ffbd20:0xc001b29020 0x3026980:0xc001b28240 0x3026d20:0xc001b282a0 0x3027800:0xc000c66360 0x3029160:0xc001b28000 0x30298a0:0xc001b283c0 0x302a720:0xc001b281e0] 0xc000126620 0} I0420 08:56:19.131425 3808 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c20360 0xc0002ed8f0 0xc0002ed960 0xc0002ed9d0 0xc0002eda40 0xc0002edb20 0xc0002edb90 0xc000c52780 0xc0002edc00 0xc0002edc70 0xc000c52820 0xc006133340 0xc0002edce0 map[0x2ff9080:0xc001b28960 0x2ff95c0:0xc001b28f60 0x2ffa2e0:0xc001b288a0 0x2ffa580:0xc001b28900 0x2ffad60:0xc001b289c0 0x2ffb000:0xc001b28a20 0x2ffb2a0:0xc001b28a80 0x2ffba80:0xc001b28fc0 0x2ffbd20:0xc001b29020 0x3026980:0xc001b28240 0x3026d20:0xc001b282a0 0x3027800:0xc000c66360 0x3029160:0xc001b28000 0x30298a0:0xc001b283c0 0x302a720:0xc001b281e0] 0xc000126620 0} I0420 08:56:19.131465 3808 gateway.go:532] Reconciling gateway with updates I0420 08:56:19.136200 3808 gateway.go:339] Spawning Conntrack Rule Check Thread I0420 08:56:19.136273 3808 default_node_network_controller.go:944] Gateway and management port readiness took 62.603184ms I0420 08:56:19.136279 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 08:56:19.136287 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0420 08:56:19.136296 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 08:56:19.136301 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0420 08:56:19.136302 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 08:56:19.136310 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0420 08:56:19.136317 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 08:56:19.136325 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 08:56:19.136325 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 08:56:19.136328 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 08:56:19.136334 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 08:56:19.136326 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0420 08:56:19.136339 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 08:56:19.136316 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 08:56:19.136345 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 08:56:19.136349 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 08:56:19.136350 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 08:56:19.136354 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 08:56:19.136360 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0420 08:56:19.136369 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 08:56:19.136338 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 08:56:19.136377 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0420 08:56:19.136383 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0420 08:56:19.136388 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0420 08:56:19.136388 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 08:56:19.136390 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 08:56:19.136395 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 08:56:19.136401 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0420 08:56:19.136397 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0420 08:56:19.136408 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 08:56:19.136410 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 08:56:19.136414 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0420 08:56:19.136399 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 08:56:19.136416 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0420 08:56:19.136422 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 08:56:19.136395 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 08:56:19.136428 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:19.136430 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 08:56:19.136435 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 08:56:19.136439 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 08:56:19.136443 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:19.136431 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 08:56:19.136444 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 08:56:19.136456 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0420 08:56:19.136462 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0420 08:56:19.136462 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0420 08:56:19.136462 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 08:56:19.136464 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0420 08:56:19.136379 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0420 08:56:19.136473 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0420 08:56:19.136476 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0420 08:56:19.136475 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 08:56:19.136478 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0420 08:56:19.136481 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0420 08:56:19.136484 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 08:56:19.136486 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0420 08:56:19.136494 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 08:56:19.136494 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 08:56:19.136479 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0420 08:56:19.136413 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0420 08:56:19.136505 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 08:56:19.136513 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 08:56:19.136514 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 08:56:19.136522 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 08:56:19.136528 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 08:56:19.136529 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 08:56:19.136530 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0420 08:56:19.136436 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0420 08:56:19.136544 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0420 08:56:19.136457 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 08:56:19.136558 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 08:56:19.136569 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 08:56:19.136340 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 08:56:19.136576 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0420 08:56:19.136539 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 08:56:19.136377 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 08:56:19.136517 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0420 08:56:19.136606 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 08:56:19.136538 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 08:56:19.136618 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0420 08:56:19.136367 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0420 08:56:19.136408 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0420 08:56:19.136560 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0420 08:56:19.136445 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 08:56:19.136665 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 08:56:19.136683 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 08:56:19.136698 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 08:56:19.136680 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0420 08:56:19.136378 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 08:56:19.136715 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 08:56:19.136419 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 08:56:19.136732 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0420 08:56:19.136469 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0420 08:56:19.136448 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 08:56:19.136412 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 08:56:19.136498 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 08:56:19.136497 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 08:56:19.136509 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 08:56:19.136820 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 08:56:19.136550 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0420 08:56:19.136839 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 30549 is already open, no action needed I0420 08:56:19.136849 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 32221 is already open, no action needed I0420 08:56:19.136633 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0420 08:56:19.136722 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0420 08:56:19.136467 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0420 08:56:19.136582 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 08:56:19.137271 3808 node_ip_handler_linux.go:160] Node IP manager is running W0420 08:56:19.173885 3808 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0420 08:56:19.174082 3808 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0420 08:56:19.174220 3808 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0420 08:56:19.184147 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0420 08:56:19.184173 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0420 08:56:19.184187 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0420 08:56:19.184196 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0420 08:56:19.184208 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0420 08:56:19.184242 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0420 08:56:19.184281 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0420 08:56:19.184292 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0420 08:56:19.184297 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0420 08:56:19.184303 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0420 08:56:19.184309 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0420 08:56:19.184314 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0420 08:56:19.184319 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0420 08:56:19.184324 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0420 08:56:19.229682 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0420 08:56:19.229711 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0420 08:56:19.229736 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0420 08:56:19.229755 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0420 08:56:19.229775 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0420 08:56:19.229791 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0420 08:56:19.229805 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0420 08:56:19.229811 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0420 08:56:19.229817 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0420 08:56:19.269648 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0420 08:56:19.269657 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0420 08:56:19.269674 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0420 08:56:19.269698 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0420 08:56:19.269709 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0420 08:56:20.196078 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-148.ec2.internal" for egress firewall I0420 08:56:20.196122 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-148.ec2.internal I0420 08:56:20.196144 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-148.ec2.internal: 43.956µs I0420 08:56:20.196165 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-148.ec2.internal in network "default" I0420 08:56:20.196179 3808 node_tracker.go:220] Node ip-10-0-2-148.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0420 08:56:20.196216 3808 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-148.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0420 08:56:20.196235 3808 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-148.ec2.internal, error: [adding or updating remote node chassis ip-10-0-2-148.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-148.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-148.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-148.ec2.internal, error parsing annotation for node ip-10-0-2-148.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0420 08:56:20.210530 3808 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-148.ec2.internal annotation: unexpected end of JSON input I0420 08:56:20.210627 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0420 08:56:20.210654 3808 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-148.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0420 08:56:20.224823 3808 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-148.ec2.internal annotation: unexpected end of JSON input I0420 08:56:20.224831 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-148.ec2.internal" for egress firewall I0420 08:56:20.224839 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0420 08:56:20.224881 3808 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-148.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0420 08:56:20.224890 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-148.ec2.internal I0420 08:56:20.224903 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-148.ec2.internal: 16.327µs E0420 08:56:20.241819 3808 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-148.ec2.internal annotation: unexpected end of JSON input E0420 08:56:20.241844 3808 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-2-148.ec2.internal annotation: unexpected end of JSON input I0420 08:56:20.241864 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0420 08:56:20.241910 3808 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-2-148.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-2-148.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-2-148.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-2-148.ec2.internal" I0420 08:56:20.254259 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0420 08:56:20.470818 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-148.ec2.internal I0420 08:56:20.470832 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-148.ec2.internal: 18.166µs I0420 08:56:20.470889 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-148.ec2.internal" for egress firewall I0420 08:56:20.470944 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:20.495760 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:20.737470 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:20.832587 3808 default_node_network_controller.go:1380] Starting node controller node sync W0420 08:56:20.832643 3808 egressip_healthcheck.go:76] Health checking using insecure connection I0420 08:56:20.832685 3808 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0420 08:56:20.844118 3808 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.509646ms I0420 08:56:20.844264 3808 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0420 08:56:20.844502 3808 egressservice_node.go:93] Setting up event handlers for Egress Services I0420 08:56:20.844620 3808 egressservice_node.go:183] Starting Egress Services Controller I0420 08:56:20.844635 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0420 08:56:20.844646 3808 shared_informer.go:356] "Caches are synced" controller="egressservices" I0420 08:56:20.844671 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0420 08:56:20.844679 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0420 08:56:20.844685 3808 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0420 08:56:20.844691 3808 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0420 08:56:20.844696 3808 egressservice_node.go:197] Repairing Egress Services I0420 08:56:20.854127 3808 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0420 08:56:20.854137 3808 external_controller.go:278] Starting Admin Policy Based Route Controller I0420 08:56:20.854353 3808 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0420 08:56:20.854364 3808 link_network_manager.go:119] Link manager is running I0420 08:56:20.854441 3808 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0420 08:56:20.854608 3808 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0420 08:56:20.855184 3808 vrf_manager.go:128] VRF manager is running I0420 08:56:20.855308 3808 node_controller_manager.go:437] Removing flows to drop GARP I0420 08:56:20.855321 3808 gateway.go:532] Reconciling gateway with updates I0420 08:56:20.880026 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 08:56:20.880045 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 08:56:20.880048 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 08:56:20.880059 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0420 08:56:20.880071 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0420 08:56:20.880082 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0420 08:56:20.880091 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 08:56:20.880094 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 08:56:20.880125 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0420 08:56:20.880128 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 08:56:20.880132 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0420 08:56:20.880136 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0420 08:56:20.880140 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 08:56:20.880151 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 08:56:20.880155 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 08:56:20.880162 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0420 08:56:20.880160 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0420 08:56:20.880168 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 08:56:20.880172 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 08:56:20.880177 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0420 08:56:20.880184 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0420 08:56:20.880186 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0420 08:56:20.880188 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 08:56:20.880191 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0420 08:56:20.880196 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 08:56:20.880201 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0420 08:56:20.880209 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0420 08:56:20.880093 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0420 08:56:20.880212 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0420 08:56:20.880218 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 08:56:20.880226 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0420 08:56:20.880229 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0420 08:56:20.880233 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0420 08:56:20.880240 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:20.880244 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0420 08:56:20.880054 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0420 08:56:20.880069 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0420 08:56:20.880227 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 08:56:20.880261 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0420 08:56:20.880265 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 08:56:20.880259 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 08:56:20.880274 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0420 08:56:20.880276 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0420 08:56:20.880274 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 08:56:20.880283 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0420 08:56:20.880244 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0420 08:56:20.880287 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0420 08:56:20.880290 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0420 08:56:20.880293 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0420 08:56:20.880116 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0420 08:56:20.880186 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 08:56:20.880345 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0420 08:56:20.880358 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0420 08:56:20.880355 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 08:56:20.880370 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0420 08:56:20.880375 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0420 08:56:20.880276 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0420 08:56:20.880186 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0420 08:56:20.880190 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 08:56:20.880398 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0420 08:56:20.880394 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0420 08:56:20.880409 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0420 08:56:20.880414 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0420 08:56:20.880172 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 08:56:20.880429 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0420 08:56:20.880203 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0420 08:56:20.880439 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0420 08:56:20.880124 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 08:56:20.880155 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0420 08:56:20.880457 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0420 08:56:20.880144 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0420 08:56:20.880466 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0420 08:56:20.880139 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0420 08:56:20.880467 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 30549 is already open, no action needed I0420 08:56:20.880484 3808 port_claim.go:71] Svc openshift-ingress/router-default: port 32221 is already open, no action needed I0420 08:56:20.880504 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 08:56:20.880026 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 08:56:20.880525 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0420 08:56:20.880536 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 08:56:20.880554 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0420 08:56:20.880536 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0420 08:56:20.880145 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0420 08:56:20.880700 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0420 08:56:20.880157 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0420 08:56:20.880715 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0420 08:56:20.880156 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0420 08:56:20.880727 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0420 08:56:20.880173 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0420 08:56:20.880791 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0420 08:56:20.880199 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0420 08:56:20.880195 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 08:56:20.880826 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0420 08:56:20.880220 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 08:56:20.880832 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0420 08:56:20.880842 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0420 08:56:20.880244 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 08:56:20.880900 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0420 08:56:20.880255 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:20.880263 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0420 08:56:20.880262 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 08:56:20.880998 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0420 08:56:20.880280 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0420 08:56:20.880280 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 08:56:20.881052 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0420 08:56:20.880293 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0420 08:56:20.880380 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0420 08:56:20.880353 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 08:56:20.880446 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0420 08:56:20.880459 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0420 08:56:20.880137 3808 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0420 08:56:20.881209 3808 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0420 08:56:20.928372 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0420 08:56:20.928400 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0420 08:56:20.928412 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0420 08:56:20.928424 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0420 08:56:20.928437 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0420 08:56:20.928446 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0420 08:56:20.928461 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0420 08:56:20.928469 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0420 08:56:20.971168 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0420 08:56:20.971180 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0420 08:56:20.971205 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0420 08:56:20.971216 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0420 08:56:20.971223 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0420 08:56:20.971229 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0420 08:56:20.971235 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0420 08:56:20.971239 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0420 08:56:21.009820 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0420 08:56:21.009844 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0420 08:56:21.009852 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0420 08:56:21.009860 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0420 08:56:21.009866 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0420 08:56:21.009874 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0420 08:56:21.009881 3808 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0420 08:56:21.069616 3808 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-bs6qh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:56:21.069644 3808 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-bs6qh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:56:22.065936 3808 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-bs6qh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:56:22.065956 3808 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-bs6qh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:56:22.765274 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0420 08:56:22.765311 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 45.035µs I0420 08:56:22.792046 3808 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-84994b6b87-r6snm] creating logical port openshift-image-registry_image-registry-84994b6b87-r6snm for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:22.792243 3808 kube.go:256] Updating pod openshift-image-registry/image-registry-84994b6b87-r6snm I0420 08:56:22.812567 3808 pod.go:62] [openshift-image-registry/image-registry-84994b6b87-r6snm] pod update took 20.376919ms I0420 08:56:22.812638 3808 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-84994b6b87-r6snm] addLogicalPort annotation time took 20.449863ms I0420 08:56:22.813940 3808 pods.go:271] [openshift-image-registry/image-registry-84994b6b87-r6snm] addLogicalPort took 21.912173ms, libovsdb time 1.006071ms I0420 08:56:22.816180 3808 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-jfhqn] creating logical port openshift-dns_dns-default-jfhqn for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:22.816383 3808 kube.go:256] Updating pod openshift-dns/dns-default-jfhqn I0420 08:56:22.834690 3808 pod.go:62] [openshift-dns/dns-default-jfhqn] pod update took 18.332731ms I0420 08:56:22.834710 3808 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-jfhqn] addLogicalPort annotation time took 18.354087ms I0420 08:56:22.835799 3808 pods.go:271] [openshift-dns/dns-default-jfhqn] addLogicalPort took 19.630624ms, libovsdb time 844.195µs I0420 08:56:22.838181 3808 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-thjfj] creating logical port openshift-ingress-canary_ingress-canary-thjfj for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:22.838286 3808 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-xz9jc for pod on switch ip-10-0-1-176.ec2.internal I0420 08:56:22.838342 3808 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-thjfj I0420 08:56:22.838404 3808 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc I0420 08:56:22.860689 3808 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc] pod update took 22.315144ms I0420 08:56:22.860709 3808 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc] addLogicalPort annotation time took 22.334814ms I0420 08:56:22.860777 3808 pod.go:62] [openshift-ingress-canary/ingress-canary-thjfj] pod update took 22.471345ms I0420 08:56:22.860838 3808 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-thjfj] addLogicalPort annotation time took 22.533616ms I0420 08:56:22.861817 3808 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc] addLogicalPort took 23.543056ms, libovsdb time 844.372µs I0420 08:56:22.862081 3808 pods.go:271] [openshift-ingress-canary/ingress-canary-thjfj] addLogicalPort took 23.916728ms, libovsdb time 1.002779ms I0420 08:56:23.935025 3808 cni.go:452] [openshift-ingress-canary/ingress-canary-thjfj c8764c42ea83d85ba0b9d7a663938e9d40af4ac19ac9bc2b4666899a73498c15 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-thjfj c8764c42ea83d85ba0b9d7a663938e9d40af4ac19ac9bc2b4666899a73498c15 network default NAD default NAD key ] I0420 08:56:23.936118 3808 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc c2768f446c6ec44838f7dde5e0c05da1bdbefd2af25da2f257ec0add8502e7a5 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc c2768f446c6ec44838f7dde5e0c05da1bdbefd2af25da2f257ec0add8502e7a5 network default NAD default NAD key ] I0420 08:56:23.984301 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-thjfj, hostIfaceName: c8764c42ea83d85, network: default, NAD default, SandboxID: "c8764c42ea83d85ba0b9d7a663938e9d40af4ac19ac9bc2b4666899a73498c15", PCI device ID: , UID: "d8663b9b-5fd5-4d09-8558-56609e68a8b0", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0420 08:56:23.984370 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-xz9jc, hostIfaceName: c2768f446c6ec44, network: default, NAD default, SandboxID: "c2768f446c6ec44838f7dde5e0c05da1bdbefd2af25da2f257ec0add8502e7a5", PCI device ID: , UID: "e624eb31-e1cf-4f19-94c3-8bcaf651dec8", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0420 08:56:24.050046 3808 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc c2768f446c6ec44838f7dde5e0c05da1bdbefd2af25da2f257ec0add8502e7a5 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-xz9jc c2768f446c6ec44838f7dde5e0c05da1bdbefd2af25da2f257ec0add8502e7a5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:74:6e:d0:67:a5\",\"name\":\"c2768f446c6ec44\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3545f956-c769-4eb3-ac2a-516fefc78055\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:56:24.050040 3808 cni.go:473] [openshift-ingress-canary/ingress-canary-thjfj c8764c42ea83d85ba0b9d7a663938e9d40af4ac19ac9bc2b4666899a73498c15 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-thjfj c8764c42ea83d85ba0b9d7a663938e9d40af4ac19ac9bc2b4666899a73498c15 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:0b:3d:ff:f7:50\",\"name\":\"c8764c42ea83d85\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e674568b-429f-417d-b31a-06fea71f9fa2\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:56:24.172701 3808 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0420 08:56:24.172709 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0420 08:56:24.172730 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 25.14µs I0420 08:56:24.172784 3808 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:25.295514 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:27.328378 3808 cni.go:452] [openshift-network-diagnostics/network-check-target-rbwxv e286a6d9c1a3ee44768577f2fdc3b02da6ace04cbd9a2a730f381ce241704aad network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-rbwxv e286a6d9c1a3ee44768577f2fdc3b02da6ace04cbd9a2a730f381ce241704aad network default NAD default NAD key ] I0420 08:56:27.349026 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-rbwxv, hostIfaceName: e286a6d9c1a3ee4, network: default, NAD default, SandboxID: "e286a6d9c1a3ee44768577f2fdc3b02da6ace04cbd9a2a730f381ce241704aad", PCI device ID: , UID: "0039b78f-ad9e-43e9-a4fe-4f1f211e2de5", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0420 08:56:27.395757 3808 cni.go:473] [openshift-network-diagnostics/network-check-target-rbwxv e286a6d9c1a3ee44768577f2fdc3b02da6ace04cbd9a2a730f381ce241704aad network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-rbwxv e286a6d9c1a3ee44768577f2fdc3b02da6ace04cbd9a2a730f381ce241704aad network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:67:2a:83:60:21\",\"name\":\"e286a6d9c1a3ee4\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a93b4eea-1016-4ffc-b421-7562285ee766\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:56:29.489019 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0420 08:56:29.489067 3808 node_tracker.go:235] Node ip-10-0-0-221.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-221.ec2.internal" I0420 08:56:29.489076 3808 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "ip-10-0-0-221.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:29.489084 3808 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:29.489129 3808 services_controller.go:574] Full service sync requested for network=default I0420 08:56:29.525252 3808 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "ip-10-0-0-221.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:29.649642 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0420 08:56:29.649699 3808 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:29.649698 3808 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "ip-10-0-0-221.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:29.649711 3808 services_controller.go:574] Full service sync requested for network=default I0420 08:56:29.650767 3808 zone_ic_handler.go:195] Time taken to create transit switch: 304.647µs I0420 08:56:29.650785 3808 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-221.ec2.internal for the network default I0420 08:56:29.651864 3808 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-221.ec2.internal" on network "default" took: 1.418722ms W0420 08:56:30.072080 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:30.582144 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0420 08:56:31.078871 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 08:56:31.203805 3808 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0420 08:56:31.203831 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0420 08:56:31.203845 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 20.594µs I0420 08:56:31.203897 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0420 08:56:31.203912 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 10.0.0.221 to PMTUD blocking rules I0420 08:56:31.203924 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0420 08:56:31.203955 3808 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:31.203974 3808 services_controller.go:574] Full service sync requested for network=default W0420 08:56:31.204231 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0420 08:56:31.204254 3808 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:31.210734 3808 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-221.ec2.internal. Updating no re-route policies I0420 08:56:31.221961 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 10.0.0.221 to PMTUD blocking rules I0420 08:56:31.221975 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0420 08:56:34.106666 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0420 08:56:34.979342 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0420 08:56:34.979375 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 40.221µs I0420 08:56:39.117445 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:39.117502 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-148.ec2.internal" for egress firewall I0420 08:56:39.117448 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-148.ec2.internal I0420 08:56:39.117531 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-148.ec2.internal: 86.403µs W0420 08:56:43.328092 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 08:56:43.942726 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-148.ec2.internal in network "default" I0420 08:56:43.942755 3808 node_tracker.go:235] Node ip-10-0-2-148.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-148.ec2.internal" I0420 08:56:43.942727 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "ip-10-0-2-148.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:43.942774 3808 node_tracker.go:169] Node ip-10-0-2-148.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:43.942786 3808 services_controller.go:574] Full service sync requested for network=default W0420 08:56:43.942978 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 08:56:44.099790 3808 default_network_controller.go:1025] Node "ip-10-0-2-148.ec2.internal" in remote zone "ip-10-0-2-148.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0420 08:56:44.099828 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-148.ec2.internal in network "default" I0420 08:56:44.099877 3808 node_tracker.go:169] Node ip-10-0-2-148.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:44.099889 3808 services_controller.go:574] Full service sync requested for network=default W0420 08:56:44.100401 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 08:56:44.100744 3808 zone_ic_handler.go:195] Time taken to create transit switch: 284.431µs I0420 08:56:44.100758 3808 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-148.ec2.internal for the network default I0420 08:56:44.101767 3808 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-148.ec2.internal" on network "default" took: 1.321446ms I0420 08:56:46.651192 3808 egressfirewall.go:723] Syncing node "ip-10-0-2-148.ec2.internal" for egress firewall I0420 08:56:46.651203 3808 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-148.ec2.internal in network "default" I0420 08:56:46.651223 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-148.ec2.internal I0420 08:56:46.651248 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-148.ec2.internal: 34.084µs I0420 08:56:46.651262 3808 node_tracker.go:169] Node ip-10-0-2-148.ec2.internal switch + router changed, syncing services in network "default" I0420 08:56:46.651278 3808 services_controller.go:574] Full service sync requested for network=default I0420 08:56:46.651203 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-148.ec2.internal", IP: 10.0.2.148 to PMTUD blocking rules I0420 08:56:46.651333 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-148.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0420 08:56:46.651744 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 08:56:46.658069 3808 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-148.ec2.internal. Updating no re-route policies I0420 08:56:46.667306 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-148.ec2.internal", IP: 10.0.2.148 to PMTUD blocking rules I0420 08:56:46.667321 3808 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-148.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0420 08:56:50.567419 3808 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-148.ec2.internal I0420 08:56:50.567455 3808 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-148.ec2.internal: 47.809µs W0420 08:56:58.270914 3808 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0420 08:57:02.084810 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0420 08:57:29.463024 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:57:29.463550 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:57:29.463582 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 08:57:29.463593 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0420 08:57:29.463612 3808 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 08:57:29.463620 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0420 08:57:30.488745 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:57:30.488764 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:57:30.488778 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:57:30.488792 3808 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390jn247 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:57:32.570862 3808 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0420 08:57:32.581814 3808 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0420 08:57:34.226689 3808 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-dsvmb for pod on switch ip-10-0-1-176.ec2.internal I0420 08:57:34.226803 3808 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb I0420 08:57:34.244618 3808 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb] pod update took 17.844184ms I0420 08:57:34.244642 3808 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb] addLogicalPort annotation time took 17.877286ms I0420 08:57:34.245940 3808 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb] addLogicalPort took 19.268548ms, libovsdb time 986.08µs I0420 08:57:34.587786 3808 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb 9ebb22094d406beb87e692b5b5d9eed009b75022821820392d8468f59a2c4e2d network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb 9ebb22094d406beb87e692b5b5d9eed009b75022821820392d8468f59a2c4e2d network default NAD default NAD key ] I0420 08:57:34.599599 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-dsvmb, hostIfaceName: 9ebb22094d406be, network: default, NAD default, SandboxID: "9ebb22094d406beb87e692b5b5d9eed009b75022821820392d8468f59a2c4e2d", PCI device ID: , UID: "3ef46be6-d6a4-4e15-a59b-41210fe5f915", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0420 08:57:34.644379 3808 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb 9ebb22094d406beb87e692b5b5d9eed009b75022821820392d8468f59a2c4e2d network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-dsvmb 9ebb22094d406beb87e692b5b5d9eed009b75022821820392d8468f59a2c4e2d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:47:76:b4:d0:f3\",\"name\":\"9ebb22094d406be\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/910467d9-470e-4648-9941-ef20ee417c0e\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 08:57:37.232403 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0420 08:57:48.240658 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0420 08:58:18.522806 3808 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5fcb75cf99-st6hs] creating logical port openshift-ingress_router-default-5fcb75cf99-st6hs for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:18.522922 3808 kube.go:256] Updating pod openshift-ingress/router-default-5fcb75cf99-st6hs I0420 08:58:18.524938 3808 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6d76f45d78-nvqgw] creating logical port openshift-ingress_router-default-6d76f45d78-nvqgw for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:18.525033 3808 kube.go:256] Updating pod openshift-ingress/router-default-6d76f45d78-nvqgw I0420 08:58:18.551131 3808 pod.go:62] [openshift-ingress/router-default-6d76f45d78-nvqgw] pod update took 26.129673ms I0420 08:58:18.551153 3808 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-6d76f45d78-nvqgw] addLogicalPort annotation time took 26.157847ms I0420 08:58:18.552240 3808 pods.go:271] [openshift-ingress/router-default-6d76f45d78-nvqgw] addLogicalPort took 27.309931ms, libovsdb time 847.088µs I0420 08:58:18.552811 3808 pod.go:62] [openshift-ingress/router-default-5fcb75cf99-st6hs] pod update took 29.915893ms I0420 08:58:18.552827 3808 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5fcb75cf99-st6hs] addLogicalPort annotation time took 29.935144ms I0420 08:58:18.553805 3808 pods.go:271] [openshift-ingress/router-default-5fcb75cf99-st6hs] addLogicalPort took 31.011899ms, libovsdb time 763.824µs W0420 08:58:21.547913 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0420 08:58:22.231198 3808 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0420 08:58:22.232030 3808 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 810.834µs I0420 08:58:22.255092 3808 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0420 08:58:22.571323 3808 namespace.go:98] [openshift-service-ca] adding namespace I0420 08:58:22.572167 3808 namespace.go:102] [openshift-service-ca] adding namespace took 821.943µs I0420 08:58:22.587915 3808 namespace.go:142] [openshift-service-ca] updating namespace W0420 08:58:29.216415 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0420 08:58:38.952007 3808 cni.go:452] [openshift-dns/dns-default-jfhqn 0754bc69e552d8b8cf13255468888d7b442b20bb6dbc554f29d546f4a9589336 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-jfhqn 0754bc69e552d8b8cf13255468888d7b442b20bb6dbc554f29d546f4a9589336 network default NAD default NAD key ] I0420 08:58:38.961612 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-jfhqn, hostIfaceName: 0754bc69e552d8b, network: default, NAD default, SandboxID: "0754bc69e552d8b8cf13255468888d7b442b20bb6dbc554f29d546f4a9589336", PCI device ID: , UID: "72ac96da-4e0f-46fc-bd45-bc9c103b69a1", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0420 08:58:39.007984 3808 cni.go:473] [openshift-dns/dns-default-jfhqn 0754bc69e552d8b8cf13255468888d7b442b20bb6dbc554f29d546f4a9589336 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-jfhqn 0754bc69e552d8b8cf13255468888d7b442b20bb6dbc554f29d546f4a9589336 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:64:6d:12:60:25\",\"name\":\"0754bc69e552d8b\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49cd8aff-6d79-4469-81ae-13550b43badd\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:58:40.953299 3808 cni.go:452] [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key ] I0420 08:58:40.964027 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-84994b6b87-r6snm, hostIfaceName: a9813effc98f446, network: default, NAD default, SandboxID: "a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a", PCI device ID: , UID: "e9ceb1c4-07b8-46e5-8ef3-304bb671d028", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0420 08:58:41.009316 3808 cni.go:473] [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:ca:c0:72:66:a0\",\"name\":\"a9813effc98f446\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/511f4eb2-fb02-4bee-a52c-244f1f62d485\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 08:58:46.681155 3808 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0420 08:58:46.681179 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0420 08:58:46.681185 3808 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0420 08:58:46.708166 3808 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0420 08:58:46.931952 3808 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-2tbtn] creating logical port openshift-insights_insights-runtime-extractor-2tbtn for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:46.932082 3808 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-2tbtn I0420 08:58:46.982238 3808 pod.go:62] [openshift-insights/insights-runtime-extractor-2tbtn] pod update took 50.19014ms I0420 08:58:46.982310 3808 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-2tbtn] addLogicalPort annotation time took 50.266389ms I0420 08:58:46.983693 3808 pods.go:271] [openshift-insights/insights-runtime-extractor-2tbtn] addLogicalPort took 51.756184ms, libovsdb time 940.841µs I0420 08:58:47.293943 3808 cni.go:452] [openshift-insights/insights-runtime-extractor-2tbtn 7083471b5f16218eedefdbbb258f160fe2425fc42c22cd49b9b7e301bd48e33f network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-2tbtn 7083471b5f16218eedefdbbb258f160fe2425fc42c22cd49b9b7e301bd48e33f network default NAD default NAD key ] I0420 08:58:47.304249 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-2tbtn, hostIfaceName: 7083471b5f16218, network: default, NAD default, SandboxID: "7083471b5f16218eedefdbbb258f160fe2425fc42c22cd49b9b7e301bd48e33f", PCI device ID: , UID: "6bd953e7-c3e7-467e-982a-2581965ef692", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0420 08:58:47.350555 3808 cni.go:473] [openshift-insights/insights-runtime-extractor-2tbtn 7083471b5f16218eedefdbbb258f160fe2425fc42c22cd49b9b7e301bd48e33f network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-2tbtn 7083471b5f16218eedefdbbb258f160fe2425fc42c22cd49b9b7e301bd48e33f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:ca:e2:43:b8:6e\",\"name\":\"7083471b5f16218\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4bb5e1bb-10e1-483e-bc83-ea42eaec8cc1\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:58:47.676403 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-84994b6b87-gq6bs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:58:47.676982 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-84994b6b87-gq6bs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:58:47.704326 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-84994b6b87-gq6bs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:58:47.704343 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-84994b6b87-gq6bs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:58:47.715685 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-84994b6b87-gq6bs I0420 08:58:47.715705 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-84994b6b87-gq6bs W0420 08:58:48.575385 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0420 08:58:48.599811 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-687d4ddf48-qptlf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:58:48.600454 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-687d4ddf48-qptlf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:58:48.613959 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-687d4ddf48-qptlf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:58:48.613976 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-687d4ddf48-qptlf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:58:48.624035 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-687d4ddf48-qptlf I0420 08:58:48.624060 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-687d4ddf48-qptlf W0420 08:58:48.655727 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0420 08:58:48.692849 3808 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0420 08:58:49.589506 3808 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0420 08:58:49.720809 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-8k9mw] creating logical port openshift-monitoring_prometheus-operator-76b589d668-8k9mw for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:49.720904 3808 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-8k9mw I0420 08:58:49.736926 3808 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-8k9mw] pod update took 16.047926ms I0420 08:58:49.736949 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-8k9mw] addLogicalPort annotation time took 16.075446ms I0420 08:58:49.738183 3808 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-8k9mw] addLogicalPort took 17.392239ms, libovsdb time 919.272µs I0420 08:58:50.080025 3808 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-8k9mw e83c14ad806eca7e9406dbf520fb6dbfd6ece8374bafd5adf3852f4dda3e4ec7 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-8k9mw e83c14ad806eca7e9406dbf520fb6dbfd6ece8374bafd5adf3852f4dda3e4ec7 network default NAD default NAD key ] I0420 08:58:50.090394 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-8k9mw, hostIfaceName: e83c14ad806eca7, network: default, NAD default, SandboxID: "e83c14ad806eca7e9406dbf520fb6dbfd6ece8374bafd5adf3852f4dda3e4ec7", PCI device ID: , UID: "b4e89bce-0283-4083-b960-93eca4be8716", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0420 08:58:50.135510 3808 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-8k9mw e83c14ad806eca7e9406dbf520fb6dbfd6ece8374bafd5adf3852f4dda3e4ec7 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-8k9mw e83c14ad806eca7e9406dbf520fb6dbfd6ece8374bafd5adf3852f4dda3e4ec7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:1e:e7:7c:30:34\",\"name\":\"e83c14ad806eca7\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7886060f-863c-4ae8-bd8e-2cf95601cdab\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:58:50.687551 3808 cni.go:452] [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key ] I0420 08:58:50.690457 3808 cni.go:452] [openshift-ingress/router-default-6d76f45d78-nvqgw 8694fab1897af0c35fcc4a5492af5d7ab0d1ca89f6a2378741b8f969b8a4fff1 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-6d76f45d78-nvqgw 8694fab1897af0c35fcc4a5492af5d7ab0d1ca89f6a2378741b8f969b8a4fff1 network default NAD default NAD key ] I0420 08:58:50.702731 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5fcb75cf99-st6hs, hostIfaceName: 41a31d416e61c95, network: default, NAD default, SandboxID: "41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d", PCI device ID: , UID: "2eddc9de-69a9-4257-99a1-b7e3c978e10c", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0420 08:58:50.704175 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6d76f45d78-nvqgw, hostIfaceName: 8694fab1897af0c, network: default, NAD default, SandboxID: "8694fab1897af0c35fcc4a5492af5d7ab0d1ca89f6a2378741b8f969b8a4fff1", PCI device ID: , UID: "85ce683c-597a-4c65-9de1-26790324ae56", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0420 08:58:50.748476 3808 cni.go:473] [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:71:e4:20:05:54\",\"name\":\"41a31d416e61c95\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f8602cf-2adf-4285-9be7-e732d0b09403\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:58:50.764031 3808 cni.go:473] [openshift-ingress/router-default-6d76f45d78-nvqgw 8694fab1897af0c35fcc4a5492af5d7ab0d1ca89f6a2378741b8f969b8a4fff1 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-6d76f45d78-nvqgw 8694fab1897af0c35fcc4a5492af5d7ab0d1ca89f6a2378741b8f969b8a4fff1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:d2:d8:df:e5:0c\",\"name\":\"8694fab1897af0c\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b062b8a-88bb-4117-b31f-b8b2a920276d\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 08:58:51.399077 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0420 08:58:51.399120 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0420 08:58:51.399130 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0420 08:58:51.438214 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.438245 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.438251 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.439238 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:51.439257 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:51.459945 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:51.459960 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:51.461918 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.461942 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.461951 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.605243 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:51.605261 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:51.606757 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.606775 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:51.606779 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:52.481888 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0420 08:58:52.481913 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0420 08:58:52.557904 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0420 08:58:52.569798 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:52.569815 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0420 08:58:52.569821 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0420 08:58:52.681937 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0420 08:58:54.155974 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-blgxz] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-blgxz for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:54.156060 3808 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-blgxz I0420 08:58:54.176952 3808 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-blgxz] pod update took 20.914996ms I0420 08:58:54.176975 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-blgxz] addLogicalPort annotation time took 20.944273ms I0420 08:58:54.178169 3808 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-blgxz] addLogicalPort took 22.211443ms, libovsdb time 880.474µs I0420 08:58:54.184231 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9] creating logical port openshift-monitoring_kube-state-metrics-6dfffbb599-cwvw9 for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:54.184336 3808 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9 I0420 08:58:54.211205 3808 pod.go:62] [openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9] pod update took 26.901876ms I0420 08:58:54.211224 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9] addLogicalPort annotation time took 26.929209ms I0420 08:58:54.212341 3808 pods.go:271] [openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9] addLogicalPort took 28.125918ms, libovsdb time 834.96µs I0420 08:58:54.512078 3808 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-blgxz 57dbb30e4a85e969921ba93edfb0ef4b0f75f465a1efab80d7cf02503a119496 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-blgxz 57dbb30e4a85e969921ba93edfb0ef4b0f75f465a1efab80d7cf02503a119496 network default NAD default NAD key ] I0420 08:58:54.522843 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-blgxz, hostIfaceName: 57dbb30e4a85e96, network: default, NAD default, SandboxID: "57dbb30e4a85e969921ba93edfb0ef4b0f75f465a1efab80d7cf02503a119496", PCI device ID: , UID: "d9c6c007-ea65-4ca9-ae0b-d4d3d3f260b0", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0420 08:58:54.541615 3808 cni.go:452] [openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9 b7f26f6843b5e70b0dd7491ad81d835e5b67daa9f26bdd530ed9d9b0f8d208ce network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9 b7f26f6843b5e70b0dd7491ad81d835e5b67daa9f26bdd530ed9d9b0f8d208ce network default NAD default NAD key ] I0420 08:58:54.552449 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-6dfffbb599-cwvw9, hostIfaceName: b7f26f6843b5e70, network: default, NAD default, SandboxID: "b7f26f6843b5e70b0dd7491ad81d835e5b67daa9f26bdd530ed9d9b0f8d208ce", PCI device ID: , UID: "efeb7288-1fa5-4aee-bfb7-2550080f5ffd", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0420 08:58:54.572091 3808 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-blgxz 57dbb30e4a85e969921ba93edfb0ef4b0f75f465a1efab80d7cf02503a119496 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-blgxz 57dbb30e4a85e969921ba93edfb0ef4b0f75f465a1efab80d7cf02503a119496 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:4f:b9:f5:c2:2a\",\"name\":\"57dbb30e4a85e96\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c5056e8-3e8b-45e2-925b-7c52aef28db6\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:58:54.597638 3808 cni.go:473] [openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9 b7f26f6843b5e70b0dd7491ad81d835e5b67daa9f26bdd530ed9d9b0f8d208ce network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-cwvw9 b7f26f6843b5e70b0dd7491ad81d835e5b67daa9f26bdd530ed9d9b0f8d208ce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:7c:1a:84:fd:74\",\"name\":\"b7f26f6843b5e70\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4382ce13-dd30-451f-bb57-698e6f803c99\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 08:58:56.619801 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0420 08:58:56.724013 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0420 08:58:58.937084 3808 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-pqdcj for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:58.937201 3808 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj I0420 08:58:58.954481 3808 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj] pod update took 17.307864ms I0420 08:58:58.954507 3808 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj] addLogicalPort annotation time took 17.337855ms I0420 08:58:58.955611 3808 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj] addLogicalPort took 18.541964ms, libovsdb time 807.21µs I0420 08:58:59.224756 3808 base_network_controller_pods.go:487] [default/openshift-console/console-6bb8bdf5dd-7g7f4] creating logical port openshift-console_console-6bb8bdf5dd-7g7f4 for pod on switch ip-10-0-1-176.ec2.internal I0420 08:58:59.224859 3808 kube.go:256] Updating pod openshift-console/console-6bb8bdf5dd-7g7f4 I0420 08:58:59.243919 3808 pod.go:62] [openshift-console/console-6bb8bdf5dd-7g7f4] pod update took 19.085699ms I0420 08:58:59.243952 3808 base_network_controller_pods.go:951] [default/openshift-console/console-6bb8bdf5dd-7g7f4] addLogicalPort annotation time took 19.122306ms I0420 08:58:59.245229 3808 pods.go:271] [openshift-console/console-6bb8bdf5dd-7g7f4] addLogicalPort took 20.488134ms, libovsdb time 895.665µs W0420 08:58:59.276213 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0420 08:58:59.296401 3808 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj 49b2ef1911cd32498c383f77336df80de9f9e9a4d305b5fdc7a2ae72edf11fb5 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj 49b2ef1911cd32498c383f77336df80de9f9e9a4d305b5fdc7a2ae72edf11fb5 network default NAD default NAD key ] I0420 08:58:59.306188 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-pqdcj, hostIfaceName: 49b2ef1911cd324, network: default, NAD default, SandboxID: "49b2ef1911cd32498c383f77336df80de9f9e9a4d305b5fdc7a2ae72edf11fb5", PCI device ID: , UID: "73ae9483-d986-473e-8889-e70dd07dea9e", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0420 08:58:59.352558 3808 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj 49b2ef1911cd32498c383f77336df80de9f9e9a4d305b5fdc7a2ae72edf11fb5 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pqdcj 49b2ef1911cd32498c383f77336df80de9f9e9a4d305b5fdc7a2ae72edf11fb5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:bf:e8:18:89:0f\",\"name\":\"49b2ef1911cd324\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/413311f9-4de9-44c1-bf6f-932d8f700252\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 08:58:59.584847 3808 cni.go:452] [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key ] I0420 08:58:59.594199 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6bb8bdf5dd-7g7f4, hostIfaceName: 207b2b09a973048, network: default, NAD default, SandboxID: "207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8", PCI device ID: , UID: "38b8a4cc-5fa4-4519-b76b-02700a6dfe4f", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0420 08:58:59.639735 3808 cni.go:473] [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:51:ee:28:60:60\",\"name\":\"207b2b09a973048\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d8afd0d2-7906-4473-9a76-547be1566ba9\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 08:59:01.491168 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0420 08:59:02.397342 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0420 08:59:02.495069 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0420 08:59:02.512686 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0420 08:59:02.660165 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 08:59:02.660186 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0420 08:59:03.652359 3808 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0420 08:59:03.724940 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 08:59:03.724954 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 08:59:03.724959 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 08:59:03.748759 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 08:59:03.748778 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 08:59:03.748783 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0420 08:59:04.686705 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 08:59:04.686732 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 08:59:04.832515 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 08:59:04.832532 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0420 08:59:05.753434 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 08:59:06.689405 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0420 08:59:08.795941 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:08.795965 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:08.795973 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:08.795978 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:09.736478 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:09.736495 3808 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:09.736500 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:09.736504 3808 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0420 08:59:10.536198 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0420 08:59:22.147946 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0420 08:59:24.578991 3808 obj_retry.go:498] Detected object openshift-console/console-7779449b58-n27mj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:59:24.579666 3808 obj_retry.go:498] Detected object openshift-console/console-7779449b58-n27mj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:59:24.856664 3808 obj_retry.go:492] Detected object openshift-console/console-7779449b58-n27mj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:59:24.856690 3808 obj_retry.go:492] Detected object openshift-console/console-7779449b58-n27mj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:59:24.859705 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7779449b58-n27mj I0420 08:59:24.859746 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7779449b58-n27mj W0420 08:59:31.834070 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0420 08:59:32.904346 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0420 08:59:34.872977 3808 cni.go:452] [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key ] I0420 08:59:34.958563 3808 cni.go:473] [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-84994b6b87-r6snm a9813effc98f446d1560b36b5a9b1de4d90ac982fa790baf314c1f36bf55559a network default NAD default NAD key default], result "{}", err I0420 08:59:35.007503 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-84994b6b87-r6snm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:59:35.007536 3808 pods.go:176] Deleting pod: openshift-image-registry/image-registry-84994b6b87-r6snm I0420 08:59:35.008663 3808 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-84994b6b87-r6snm, ips: 10.128.0.5 I0420 08:59:35.008687 3808 obj_retry.go:498] Detected object openshift-image-registry/image-registry-84994b6b87-r6snm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:59:35.618418 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-84994b6b87-r6snm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:59:35.618444 3808 obj_retry.go:492] Detected object openshift-image-registry/image-registry-84994b6b87-r6snm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:59:35.624822 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-84994b6b87-r6snm I0420 08:59:35.624850 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-84994b6b87-r6snm I0420 08:59:35.888004 3808 obj_retry.go:498] Detected object openshift-console/console-7779449b58-cdm6j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 08:59:35.888704 3808 obj_retry.go:498] Detected object openshift-console/console-7779449b58-cdm6j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 08:59:36.837221 3808 obj_retry.go:492] Detected object openshift-console/console-7779449b58-cdm6j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:59:36.837244 3808 obj_retry.go:492] Detected object openshift-console/console-7779449b58-cdm6j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 08:59:36.843748 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7779449b58-cdm6j I0420 08:59:36.843784 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7779449b58-cdm6j W0420 08:59:57.011725 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0420 09:00:00.198905 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29611260-zznn4 for pod on switch ip-10-0-1-176.ec2.internal I0420 09:00:00.199019 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 I0420 09:00:00.211994 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4] pod update took 13.00138ms I0420 09:00:00.212041 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4] addLogicalPort annotation time took 13.055136ms I0420 09:00:00.213474 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4] addLogicalPort took 14.589261ms, libovsdb time 1.055784ms I0420 09:00:00.572287 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key ] I0420 09:00:00.581701 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29611260-zznn4, hostIfaceName: 1b9f615bd41bd95, network: default, NAD default, SandboxID: "1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32", PCI device ID: , UID: "402e0bbf-22e3-41fc-90dc-ea35753b32be", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0420 09:00:00.629043 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:14:1e:34:8e:67\",\"name\":\"1b9f615bd41bd95\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4694f1ef-e530-462e-acba-f61939137866\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:00:04.741727 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key ] I0420 09:00:04.824060 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 1b9f615bd41bd95e2426cbed7e32be646893f8d0a440f9ef2b1d81049f74ec32 network default NAD default NAD key default], result "{}", err I0420 09:00:04.863432 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:04.863462 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 I0420 09:00:04.864675 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4, ips: 10.128.0.18 I0420 09:00:04.864702 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:05.050488 3808 cni.go:452] [openshift-multus/network-metrics-daemon-qgnzf 7b18410abb709a9142fa6e400209db585c4085765d450e39bd04e8fd1c231922 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-qgnzf 7b18410abb709a9142fa6e400209db585c4085765d450e39bd04e8fd1c231922 network default NAD default NAD key ] I0420 09:00:05.060201 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-qgnzf, hostIfaceName: 7b18410abb709a9, network: default, NAD default, SandboxID: "7b18410abb709a9142fa6e400209db585c4085765d450e39bd04e8fd1c231922", PCI device ID: , UID: "f3293dcc-cbc2-497d-9294-425be7b89334", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0420 09:00:05.105634 3808 cni.go:473] [openshift-multus/network-metrics-daemon-qgnzf 7b18410abb709a9142fa6e400209db585c4085765d450e39bd04e8fd1c231922 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-qgnzf 7b18410abb709a9142fa6e400209db585c4085765d450e39bd04e8fd1c231922 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:75:dc:e5:0c:0e\",\"name\":\"7b18410abb709a9\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8597345f-25e4-41ae-b525-ab6f3112eeec\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:00:05.750225 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:05.750253 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:08.206879 3808 cni.go:452] [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key ] I0420 09:00:08.307161 3808 cni.go:473] [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-5fcb75cf99-st6hs 41a31d416e61c95524245683e7bc4eb17f44e17cf323c56739fec1a6b54e679d network default NAD default NAD key default], result "{}", err I0420 09:00:08.353131 3808 obj_retry.go:498] Detected object openshift-ingress/router-default-5fcb75cf99-st6hs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:08.353162 3808 pods.go:176] Deleting pod: openshift-ingress/router-default-5fcb75cf99-st6hs I0420 09:00:08.354308 3808 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5fcb75cf99-st6hs, ips: 10.128.0.10 I0420 09:00:08.354337 3808 obj_retry.go:498] Detected object openshift-ingress/router-default-5fcb75cf99-st6hs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:08.753176 3808 obj_retry.go:492] Detected object openshift-ingress/router-default-5fcb75cf99-st6hs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:08.753282 3808 obj_retry.go:492] Detected object openshift-ingress/router-default-5fcb75cf99-st6hs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:08.760473 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5fcb75cf99-st6hs I0420 09:00:08.760506 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5fcb75cf99-st6hs I0420 09:00:16.080386 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:16.081159 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:16.403057 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:16.403082 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:16.409749 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0420 09:00:16.409778 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0420 09:00:20.449776 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:20.450455 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:21.147936 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:21.147961 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:21.154066 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0420 09:00:21.154119 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0420 09:00:47.784665 3808 obj_retry.go:498] Detected object openshift-console/console-6bb8bdf5dd-qp4gz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:47.785208 3808 obj_retry.go:498] Detected object openshift-console/console-6bb8bdf5dd-qp4gz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:48.172588 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:48.173155 3808 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:48.282561 3808 obj_retry.go:492] Detected object openshift-console/console-6bb8bdf5dd-qp4gz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:48.282583 3808 obj_retry.go:492] Detected object openshift-console/console-6bb8bdf5dd-qp4gz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:48.288390 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bb8bdf5dd-qp4gz I0420 09:00:48.288421 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bb8bdf5dd-qp4gz I0420 09:00:49.158929 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:49.158949 3808 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:49.165231 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0420 09:00:49.165262 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0420 09:00:58.295218 3808 cni.go:452] [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key ] I0420 09:00:58.376911 3808 cni.go:473] [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6bb8bdf5dd-7g7f4 207b2b09a97304834fa6d0b7f8848681610afcad1f6029db7c26503b79145cd8 network default NAD default NAD key default], result "{}", err I0420 09:00:58.425238 3808 obj_retry.go:498] Detected object openshift-console/console-6bb8bdf5dd-7g7f4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:58.425265 3808 pods.go:176] Deleting pod: openshift-console/console-6bb8bdf5dd-7g7f4 I0420 09:00:58.426459 3808 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6bb8bdf5dd-7g7f4, ips: 10.128.0.17 I0420 09:00:58.426488 3808 obj_retry.go:498] Detected object openshift-console/console-6bb8bdf5dd-7g7f4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:00:58.943855 3808 obj_retry.go:492] Detected object openshift-console/console-6bb8bdf5dd-7g7f4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:58.943877 3808 obj_retry.go:492] Detected object openshift-console/console-6bb8bdf5dd-7g7f4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:00:58.949742 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bb8bdf5dd-7g7f4 I0420 09:00:58.949766 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bb8bdf5dd-7g7f4 I0420 09:01:23.802052 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:01:23.802653 3808 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:01:24.322763 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:01:24.322797 3808 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:01:24.329773 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0420 09:01:24.329791 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0420 09:01:24.760027 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=79 I0420 09:01:27.185639 3808 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0420 09:01:33.883505 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0420 09:01:36.861552 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0420 09:01:41.764404 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0420 09:01:57.845088 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0420 09:02:13.765216 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0420 09:02:15.761290 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0420 09:02:24.872543 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0420 09:02:26.856183 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0420 09:02:34.546223 3808 base_network_controller_pods.go:487] [default/openshift-console/console-7894677dbd-4t6bp] creating logical port openshift-console_console-7894677dbd-4t6bp for pod on switch ip-10-0-1-176.ec2.internal I0420 09:02:34.546377 3808 kube.go:256] Updating pod openshift-console/console-7894677dbd-4t6bp I0420 09:02:34.569082 3808 pod.go:62] [openshift-console/console-7894677dbd-4t6bp] pod update took 22.741012ms I0420 09:02:34.569153 3808 base_network_controller_pods.go:951] [default/openshift-console/console-7894677dbd-4t6bp] addLogicalPort annotation time took 22.813084ms I0420 09:02:34.570624 3808 pods.go:271] [openshift-console/console-7894677dbd-4t6bp] addLogicalPort took 24.420214ms, libovsdb time 1.114136ms I0420 09:02:34.905147 3808 cni.go:452] [openshift-console/console-7894677dbd-4t6bp 8b75a18cd8d75cee1b618122485f11a2cbb0acae79e2bf57e1971c4bcad1b9e4 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7894677dbd-4t6bp 8b75a18cd8d75cee1b618122485f11a2cbb0acae79e2bf57e1971c4bcad1b9e4 network default NAD default NAD key ] I0420 09:02:34.914506 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7894677dbd-4t6bp, hostIfaceName: 8b75a18cd8d75ce, network: default, NAD default, SandboxID: "8b75a18cd8d75cee1b618122485f11a2cbb0acae79e2bf57e1971c4bcad1b9e4", PCI device ID: , UID: "17a72468-509c-4660-ae84-369f9a1c7add", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0420 09:02:34.962397 3808 cni.go:473] [openshift-console/console-7894677dbd-4t6bp 8b75a18cd8d75cee1b618122485f11a2cbb0acae79e2bf57e1971c4bcad1b9e4 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7894677dbd-4t6bp 8b75a18cd8d75cee1b618122485f11a2cbb0acae79e2bf57e1971c4bcad1b9e4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:9e:f5:13:88:29\",\"name\":\"8b75a18cd8d75ce\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0162f4ea-3794-4b8a-ac9f-b1def6d40916\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 09:02:45.331995 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0420 09:02:59.860455 3808 obj_retry.go:498] Detected object openshift-console/console-5bcd6df69c-bdljt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:02:59.861126 3808 obj_retry.go:498] Detected object openshift-console/console-5bcd6df69c-bdljt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:03:00.780889 3808 obj_retry.go:492] Detected object openshift-console/console-5bcd6df69c-bdljt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:03:00.780918 3808 obj_retry.go:492] Detected object openshift-console/console-5bcd6df69c-bdljt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:03:00.786904 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bcd6df69c-bdljt I0420 09:03:00.786930 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bcd6df69c-bdljt I0420 09:03:10.641223 3808 obj_retry.go:498] Detected object openshift-console/console-5bcd6df69c-wcdg5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:03:10.641904 3808 obj_retry.go:498] Detected object openshift-console/console-5bcd6df69c-wcdg5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:03:10.925307 3808 obj_retry.go:492] Detected object openshift-console/console-5bcd6df69c-wcdg5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:03:10.925328 3808 obj_retry.go:492] Detected object openshift-console/console-5bcd6df69c-wcdg5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:03:10.937489 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bcd6df69c-wcdg5 I0420 09:03:10.937512 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bcd6df69c-wcdg5 I0420 09:03:31.770916 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=661 I0420 09:03:40.762558 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=145 I0420 09:04:37.877893 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0420 09:04:53.883958 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0420 09:05:17.866741 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0420 09:05:19.850876 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0420 09:05:36.844841 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0420 09:07:01.714963 3808 namespace.go:98] [tsf] adding namespace I0420 09:07:01.715985 3808 namespace.go:102] [tsf] adding namespace took 994.587µs I0420 09:07:01.728705 3808 namespace.go:142] [tsf] updating namespace I0420 09:07:01.739773 3808 namespace.go:142] [tsf] updating namespace I0420 09:07:05.642063 3808 namespace.go:98] [konflux-ui] adding namespace I0420 09:07:05.642260 3808 namespace.go:98] [rhbk-operator] adding namespace I0420 09:07:05.642390 3808 namespace.go:98] [konflux-operator] adding namespace I0420 09:07:05.642551 3808 namespace.go:98] [tsf-keycloak] adding namespace I0420 09:07:05.643187 3808 namespace.go:102] [konflux-ui] adding namespace took 1.10071ms I0420 09:07:05.643990 3808 namespace.go:102] [rhbk-operator] adding namespace took 1.709977ms I0420 09:07:05.644749 3808 namespace.go:102] [konflux-operator] adding namespace took 2.34322ms I0420 09:07:05.645207 3808 namespace.go:98] [tsf-tas] adding namespace I0420 09:07:05.645574 3808 namespace.go:102] [tsf-keycloak] adding namespace took 3.003745ms I0420 09:07:05.646288 3808 namespace.go:102] [tsf-tas] adding namespace took 1.06264ms I0420 09:07:05.646805 3808 namespace.go:98] [cert-manager-operator] adding namespace I0420 09:07:05.646923 3808 namespace.go:98] [rhtpa-operator] adding namespace I0420 09:07:05.647066 3808 namespace.go:98] [tsf-tpa] adding namespace I0420 09:07:05.647620 3808 namespace.go:102] [cert-manager-operator] adding namespace took 796.366µs I0420 09:07:05.648325 3808 namespace.go:102] [rhtpa-operator] adding namespace took 1.384925ms I0420 09:07:05.649009 3808 namespace.go:102] [tsf-tpa] adding namespace took 1.924015ms I0420 09:07:05.655863 3808 namespace.go:142] [konflux-ui] updating namespace I0420 09:07:05.666286 3808 namespace.go:142] [konflux-ui] updating namespace I0420 09:07:05.674300 3808 namespace.go:142] [rhbk-operator] updating namespace I0420 09:07:05.691191 3808 namespace.go:142] [konflux-operator] updating namespace I0420 09:07:05.696975 3808 namespace.go:142] [rhbk-operator] updating namespace I0420 09:07:05.702872 3808 namespace.go:142] [tsf-tas] updating namespace I0420 09:07:05.710681 3808 namespace.go:142] [konflux-operator] updating namespace I0420 09:07:05.710812 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:05.714445 3808 namespace.go:142] [rhtpa-operator] updating namespace I0420 09:07:05.714501 3808 namespace.go:142] [cert-manager-operator] updating namespace I0420 09:07:05.729049 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:05.729184 3808 namespace.go:142] [konflux-operator] updating namespace I0420 09:07:05.729351 3808 namespace.go:142] [konflux-ui] updating namespace I0420 09:07:05.740214 3808 namespace.go:142] [tsf-tas] updating namespace I0420 09:07:05.753198 3808 namespace.go:142] [tsf-tas] updating namespace I0420 09:07:05.754226 3808 namespace.go:142] [cert-manager-operator] updating namespace I0420 09:07:05.776593 3808 namespace.go:142] [rhtpa-operator] updating namespace I0420 09:07:05.776695 3808 namespace.go:142] [cert-manager-operator] updating namespace I0420 09:07:05.793018 3808 namespace.go:142] [rhtpa-operator] updating namespace I0420 09:07:05.802639 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:05.814284 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:05.818936 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:05.831774 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:05.856146 3808 namespace.go:142] [rhbk-operator] updating namespace I0420 09:07:06.796658 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:06.819185 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:06.989842 3808 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-176.ec2.internal I0420 09:07:06.990004 3808 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0420 09:07:07.023317 3808 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 33.36ms I0420 09:07:07.023344 3808 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 33.392314ms I0420 09:07:07.024799 3808 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 34.977515ms, libovsdb time 971.844µs I0420 09:07:07.369361 3808 cni.go:452] [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key ] I0420 09:07:07.378527 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 482bd21c9bebe79, network: default, NAD default, SandboxID: "482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a", PCI device ID: , UID: "f2482433-e1dd-451b-abee-03bac608b47e", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0420 09:07:07.424075 3808 cni.go:473] [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:70:2a:fd:87:55\",\"name\":\"482bd21c9bebe79\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e72d5064-0af3-4686-936d-7a6b8da1ecea\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:07:10.865233 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0420 09:07:12.975993 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:12.976647 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:12.976683 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:12.976693 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 09:07:12.976707 3808 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:12.976713 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 09:07:13.054607 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:13.055166 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:13.055196 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:13.055207 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 09:07:13.055224 3808 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:13.055230 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 09:07:13.999490 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:13.999514 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:13.999526 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:13.999541 3808 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:14.077214 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:14.077242 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:14.077262 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:14.077279 3808 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568585zq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:16.013566 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:16.014137 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:16.014162 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:16.014170 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 09:07:16.014181 3808 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:16.014188 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 09:07:17.042087 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:17.042640 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:17.042662 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:17.042671 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 09:07:17.042687 3808 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:17.042693 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 09:07:17.048076 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:17.048140 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:17.048163 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:17.048183 3808 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88eceqfcs4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:18.028603 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:18.029135 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:07:18.029164 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:18.029175 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0420 09:07:18.029192 3808 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0420 09:07:18.029199 3808 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0420 09:07:18.059978 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:18.059997 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:18.060006 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:18.060019 3808 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7bchcx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:19.054838 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:19.054864 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:19.054885 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:19.054899 3808 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:07:29.521536 3808 namespace.go:142] [openshift-operators] updating namespace I0420 09:07:29.539897 3808 namespace.go:142] [openshift-operators] updating namespace W0420 09:07:31.066529 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 09:07:32.306004 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 09:07:32.321471 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 09:07:32.323873 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 09:07:32.339907 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0420 09:07:33.019396 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0420 09:07:38.292212 3808 namespace.go:142] [openshift-operators] updating namespace I0420 09:07:38.678772 3808 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-h9dl2 for pod on switch ip-10-0-1-176.ec2.internal I0420 09:07:38.678873 3808 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2 I0420 09:07:38.699917 3808 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2] pod update took 21.066687ms I0420 09:07:38.699943 3808 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2] addLogicalPort annotation time took 21.096089ms I0420 09:07:38.701014 3808 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2] addLogicalPort took 22.256205ms, libovsdb time 782.534µs I0420 09:07:39.039557 3808 cni.go:452] [openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2 11ffd3792358857ca68dd6c11afc6b053103953c2f7a560ae51ca64d6daca2ca network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2 11ffd3792358857ca68dd6c11afc6b053103953c2f7a560ae51ca64d6daca2ca network default NAD default NAD key ] I0420 09:07:39.049061 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-h9dl2, hostIfaceName: 11ffd3792358857, network: default, NAD default, SandboxID: "11ffd3792358857ca68dd6c11afc6b053103953c2f7a560ae51ca64d6daca2ca", PCI device ID: , UID: "f2591f1b-6700-455d-a156-323e5094f9f9", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] W0420 09:07:39.088346 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0420 09:07:39.095038 3808 cni.go:473] [openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2 11ffd3792358857ca68dd6c11afc6b053103953c2f7a560ae51ca64d6daca2ca network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-h9dl2 11ffd3792358857ca68dd6c11afc6b053103953c2f7a560ae51ca64d6daca2ca network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:68:00:3c:0f:25\",\"name\":\"11ffd3792358857\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/342561db-9ff1-444d-8b70-28263af0a3fb\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 09:07:39.101595 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0420 09:07:39.116203 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0420 09:07:39.123608 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0420 09:07:43.876706 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0420 09:07:49.019898 3808 namespace.go:98] [trusted-artifact-signer] adding namespace I0420 09:07:49.020676 3808 namespace.go:102] [trusted-artifact-signer] adding namespace took 760.736µs I0420 09:07:49.092220 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 09:07:49.116779 3808 namespace.go:142] [trusted-artifact-signer] updating namespace W0420 09:07:49.265365 3808 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0420 09:07:50.768959 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=50 I0420 09:07:53.549343 3808 namespace.go:98] [openshift-pipelines] adding namespace I0420 09:07:53.550213 3808 namespace.go:102] [openshift-pipelines] adding namespace took 849.669µs I0420 09:07:53.569833 3808 namespace.go:142] [openshift-pipelines] updating namespace I0420 09:07:54.221780 3808 namespace.go:142] [cert-manager-operator] updating namespace I0420 09:07:54.626625 3808 namespace.go:142] [konflux-operator] updating namespace I0420 09:07:54.827072 3808 namespace.go:142] [konflux-ui] updating namespace I0420 09:07:55.037011 3808 namespace.go:142] [open-cluster-management-2ppph9ruf6k5ml3crseffj0vunji1dnc] updating namespace I0420 09:07:55.451021 3808 namespace.go:142] [rhbk-operator] updating namespace I0420 09:07:55.642793 3808 namespace.go:142] [rhtpa-operator] updating namespace I0420 09:07:55.841298 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 09:07:56.041416 3808 namespace.go:142] [tsf] updating namespace I0420 09:07:56.242533 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:56.442439 3808 namespace.go:142] [tsf-tas] updating namespace I0420 09:07:56.644020 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:56.770945 3808 namespace.go:142] [cert-manager-operator] updating namespace I0420 09:07:56.779973 3808 namespace.go:142] [default] updating namespace I0420 09:07:56.789766 3808 namespace.go:142] [konflux-operator] updating namespace I0420 09:07:56.799149 3808 namespace.go:142] [konflux-ui] updating namespace I0420 09:07:56.809920 3808 namespace.go:142] [open-cluster-management-2ppph9ruf6k5ml3crseffj0vunji1dnc] updating namespace I0420 09:07:56.817334 3808 namespace.go:142] [openshift] updating namespace I0420 09:07:56.828132 3808 namespace.go:142] [rhbk-operator] updating namespace I0420 09:07:56.838479 3808 namespace.go:142] [rhtpa-operator] updating namespace I0420 09:07:56.849332 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 09:07:56.857689 3808 namespace.go:142] [tsf] updating namespace I0420 09:07:56.867919 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:56.883980 3808 namespace.go:142] [tsf-tas] updating namespace I0420 09:07:56.905765 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:57.005280 3808 namespace.go:142] [cert-manager-operator] updating namespace I0420 09:07:57.102899 3808 namespace.go:142] [default] updating namespace I0420 09:07:57.206388 3808 namespace.go:142] [konflux-operator] updating namespace I0420 09:07:57.304547 3808 namespace.go:142] [konflux-ui] updating namespace I0420 09:07:57.404596 3808 namespace.go:142] [open-cluster-management-2ppph9ruf6k5ml3crseffj0vunji1dnc] updating namespace I0420 09:07:57.504115 3808 namespace.go:142] [openshift] updating namespace I0420 09:07:57.607029 3808 namespace.go:142] [rhbk-operator] updating namespace I0420 09:07:57.707133 3808 namespace.go:142] [rhtpa-operator] updating namespace I0420 09:07:57.804590 3808 namespace.go:142] [trusted-artifact-signer] updating namespace I0420 09:07:57.903639 3808 namespace.go:142] [tsf] updating namespace I0420 09:07:58.005586 3808 namespace.go:142] [tsf-keycloak] updating namespace I0420 09:07:58.105052 3808 namespace.go:142] [tsf-tas] updating namespace I0420 09:07:58.206078 3808 namespace.go:142] [tsf-tpa] updating namespace I0420 09:07:58.502591 3808 namespace.go:142] [openshift-pipelines] updating namespace I0420 09:07:58.859898 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 W0420 09:08:13.285658 3808 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 09:08:13.285676 3808 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 09:08:13.285682 3808 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 09:08:13.285686 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0420 09:08:13.319004 3808 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0420 09:08:13.319022 3808 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0420 09:08:13.319029 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0420 09:08:13.392756 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0420 09:08:13.392770 3808 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0420 09:08:13.392773 3808 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0420 09:08:31.099532 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-7zmzk for pod on switch ip-10-0-1-176.ec2.internal I0420 09:08:31.099611 3808 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk I0420 09:08:31.121973 3808 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk] pod update took 22.380395ms I0420 09:08:31.122058 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk] addLogicalPort annotation time took 22.469153ms I0420 09:08:31.123524 3808 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk] addLogicalPort took 24.001277ms, libovsdb time 1.008965ms I0420 09:08:31.456703 3808 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk 58dcf56fc95b4947844f5cfb12f164b6438ea8a4ba3c32b27c932abc98e95fc5 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk 58dcf56fc95b4947844f5cfb12f164b6438ea8a4ba3c32b27c932abc98e95fc5 network default NAD default NAD key ] I0420 09:08:31.465669 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-7zmzk, hostIfaceName: 58dcf56fc95b494, network: default, NAD default, SandboxID: "58dcf56fc95b4947844f5cfb12f164b6438ea8a4ba3c32b27c932abc98e95fc5", PCI device ID: , UID: "d9097f9d-e955-4cfb-8a45-4b7958d26300", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0420 09:08:31.511704 3808 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk 58dcf56fc95b4947844f5cfb12f164b6438ea8a4ba3c32b27c932abc98e95fc5 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-7zmzk 58dcf56fc95b4947844f5cfb12f164b6438ea8a4ba3c32b27c932abc98e95fc5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:ee:ec:32:f1:c1\",\"name\":\"58dcf56fc95b494\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35a11808-6275-4ed1-bcf6-63d72574bcca\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:08:32.775267 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=160 W0420 09:08:33.645457 3808 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0420 09:08:33.886974 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0420 09:08:40.764251 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=39 W0420 09:08:44.656433 3808 util.go:959] Failed to build node endpoints for node ip-10-0-1-176.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0420 09:08:46.397139 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-lndz6 for pod on switch ip-10-0-1-176.ec2.internal I0420 09:08:46.397282 3808 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6 I0420 09:08:46.413772 3808 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6] pod update took 16.514499ms I0420 09:08:46.413802 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6] addLogicalPort annotation time took 16.550581ms I0420 09:08:46.415180 3808 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6] addLogicalPort took 18.085142ms, libovsdb time 918.819µs I0420 09:08:46.753038 3808 cni.go:452] [openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6 6b2588a65cb7c34aa1b4b456746319ce9221860b513cecf8d0c7f3cbda1432ce network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6 6b2588a65cb7c34aa1b4b456746319ce9221860b513cecf8d0c7f3cbda1432ce network default NAD default NAD key ] I0420 09:08:46.762644 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-lndz6, hostIfaceName: 6b2588a65cb7c34, network: default, NAD default, SandboxID: "6b2588a65cb7c34aa1b4b456746319ce9221860b513cecf8d0c7f3cbda1432ce", PCI device ID: , UID: "50684bdd-3b39-4219-a634-fe661f537848", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0420 09:08:46.807882 3808 cni.go:473] [openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6 6b2588a65cb7c34aa1b4b456746319ce9221860b513cecf8d0c7f3cbda1432ce network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-lndz6 6b2588a65cb7c34aa1b4b456746319ce9221860b513cecf8d0c7f3cbda1432ce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:6c:06:21:3b:53\",\"name\":\"6b2588a65cb7c34\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9e96e87-8d27-4623-a7ed-13afd76a0d9c\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:08:50.560133 3808 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0420 09:09:06.041544 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-xmnmm for pod on switch ip-10-0-1-176.ec2.internal I0420 09:09:06.041670 3808 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm I0420 09:09:06.057239 3808 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm] pod update took 15.5885ms I0420 09:09:06.057261 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm] addLogicalPort annotation time took 15.615918ms I0420 09:09:06.058389 3808 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm] addLogicalPort took 16.86053ms, libovsdb time 837.936µs I0420 09:09:06.398249 3808 cni.go:452] [openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm b8d48bc9108bdffb7f08ce0ed340dc62a9a240280a21099f01a01cf969a05bd6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm b8d48bc9108bdffb7f08ce0ed340dc62a9a240280a21099f01a01cf969a05bd6 network default NAD default NAD key ] I0420 09:09:06.407439 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-xmnmm, hostIfaceName: b8d48bc9108bdff, network: default, NAD default, SandboxID: "b8d48bc9108bdffb7f08ce0ed340dc62a9a240280a21099f01a01cf969a05bd6", PCI device ID: , UID: "64348863-5f1e-401f-90bd-5783e8c0bc27", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0420 09:09:06.456253 3808 cni.go:473] [openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm b8d48bc9108bdffb7f08ce0ed340dc62a9a240280a21099f01a01cf969a05bd6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-xmnmm b8d48bc9108bdffb7f08ce0ed340dc62a9a240280a21099f01a01cf969a05bd6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:4f:94:6f:0e:e7\",\"name\":\"b8d48bc9108bdff\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/44247d30-2853-4777-8037-e4ca2d13ee40\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 09:09:10.568398 3808 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0420 09:09:10.568422 3808 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0420 09:09:10.568429 3808 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0420 09:09:22.609265 3808 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-m2ld9] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-m2ld9 for pod on switch ip-10-0-1-176.ec2.internal I0420 09:09:22.609372 3808 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-m2ld9 I0420 09:09:22.624058 3808 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-m2ld9] pod update took 14.710968ms I0420 09:09:22.624142 3808 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-m2ld9] addLogicalPort annotation time took 14.802343ms I0420 09:09:22.625510 3808 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-m2ld9] addLogicalPort took 16.26237ms, libovsdb time 971.481µs I0420 09:09:22.969397 3808 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-m2ld9 8676919c950bf0e6a9f5642f164b9ec1ac859ac1364a7794a31f09414b8dfc5e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-m2ld9 8676919c950bf0e6a9f5642f164b9ec1ac859ac1364a7794a31f09414b8dfc5e network default NAD default NAD key ] I0420 09:09:22.978376 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-m2ld9, hostIfaceName: 8676919c950bf0e, network: default, NAD default, SandboxID: "8676919c950bf0e6a9f5642f164b9ec1ac859ac1364a7794a31f09414b8dfc5e", PCI device ID: , UID: "b2b97044-d7b7-42bd-9c39-76f6cbce20cf", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0420 09:09:23.025485 3808 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-m2ld9 8676919c950bf0e6a9f5642f164b9ec1ac859ac1364a7794a31f09414b8dfc5e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-m2ld9 8676919c950bf0e6a9f5642f164b9ec1ac859ac1364a7794a31f09414b8dfc5e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:d6:1a:48:76:7e\",\"name\":\"8676919c950bf0e\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c07f9d75-dec2-4cd7-8ed6-2f51c5e86724\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0420 09:09:43.764822 3808 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0420 09:09:44.766573 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=63 I0420 09:09:53.765494 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0420 09:09:58.768529 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=44 I0420 09:10:00.175563 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29611270-w5pft for pod on switch ip-10-0-1-176.ec2.internal I0420 09:10:00.175679 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft I0420 09:10:00.192050 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft] pod update took 16.400038ms I0420 09:10:00.192074 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft] addLogicalPort annotation time took 16.430464ms I0420 09:10:00.193436 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft] addLogicalPort took 17.893329ms, libovsdb time 967.246µs I0420 09:10:00.547260 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key ] I0420 09:10:00.557307 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29611270-w5pft, hostIfaceName: e8a9062d9e3e6a2, network: default, NAD default, SandboxID: "e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786", PCI device ID: , UID: "720a1b47-bb1f-47f7-ac85-ed549d254256", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0420 09:10:00.606114 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:ca:46:a2:f9:0a\",\"name\":\"e8a9062d9e3e6a2\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e1782b5b-c0d2-42d2-b007-e58e7eead109\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:10:04.073254 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key ] I0420 09:10:04.148090 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft e8a9062d9e3e6a21df474fe8679deaa8431448243e8f10c15fe18d76fa828786 network default NAD default NAD key default], result "{}", err I0420 09:10:04.189557 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:10:04.189589 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft I0420 09:10:04.190689 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft, ips: 10.128.0.26 I0420 09:10:04.190716 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:10:05.079058 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:10:05.079090 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611270-w5pft of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:11:01.882384 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0420 09:11:10.848964 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0420 09:11:29.871074 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0420 09:11:29.888058 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0420 09:11:48.854369 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0420 09:12:52.202503 3808 namespace.go:142] [openshift-pipelines] updating namespace I0420 09:14:18.768306 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=39 I0420 09:14:35.848492 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0420 09:14:36.890891 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0420 09:14:41.884026 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0420 09:15:38.773300 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=49 I0420 09:16:01.870051 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0420 09:16:14.864186 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0420 09:16:36.886546 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0420 09:16:44.779166 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=87 I0420 09:16:57.891773 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0420 09:17:21.853846 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0420 09:17:32.857712 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0420 09:17:37.768888 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0420 09:18:00.176571 3808 cni.go:452] [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key ] I0420 09:18:00.248709 3808 cni.go:473] [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions 482bd21c9bebe795481d339d1287ed32454ad27aa94c446886b1ae3565bcf21a network default NAD default NAD key default], result "{}", err I0420 09:18:00.301991 3808 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:18:00.302036 3808 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0420 09:18:00.303207 3808 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.20 I0420 09:18:00.303231 3808 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:18:07.773024 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0420 09:18:35.770513 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=12 I0420 09:18:47.874896 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0420 09:19:00.436831 3808 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:19:00.436877 3808 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:19:00.442787 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0420 09:19:00.442821 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0420 09:19:00.572550 3808 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-176.ec2.internal I0420 09:19:00.572681 3808 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0420 09:19:00.590676 3808 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 18.024542ms I0420 09:19:00.590702 3808 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 18.055159ms I0420 09:19:00.592057 3808 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 19.525745ms, libovsdb time 1.007583ms I0420 09:19:00.935195 3808 cni.go:452] [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key ] I0420 09:19:00.945429 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: c77d75a690726ae, network: default, NAD default, SandboxID: "c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481", PCI device ID: , UID: "0bce039f-6ba2-44cc-8295-e8357560e4e3", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0420 09:19:00.994937 3808 cni.go:473] [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:1e:cd:09:52:c2\",\"name\":\"c77d75a690726ae\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13473a39-f477-4c8d-8472-5280e50b7896\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:20:00.191470 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29611280-72v6x for pod on switch ip-10-0-1-176.ec2.internal I0420 09:20:00.191602 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x I0420 09:20:00.205433 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x] pod update took 13.856019ms I0420 09:20:00.205464 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x] addLogicalPort annotation time took 13.894576ms I0420 09:20:00.206904 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x] addLogicalPort took 15.45032ms, libovsdb time 1.080255ms I0420 09:20:00.559556 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key ] I0420 09:20:00.569679 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29611280-72v6x, hostIfaceName: 367cc29b4657129, network: default, NAD default, SandboxID: "367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a", PCI device ID: , UID: "4e18f4e9-2342-435e-8ef5-3a627adb6f84", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0420 09:20:00.618156 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:26:b0:8d:64:fe\",\"name\":\"367cc29b4657129\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f4b233db-9c8f-4a57-9bcc-51a412a4daae\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:20:03.754803 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key ] I0420 09:20:03.832838 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x 367cc29b4657129443dce2b92d813b6db5de8dae5caf4121a45c294c76b7240a network default NAD default NAD key default], result "{}", err I0420 09:20:03.878920 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:20:03.879036 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x I0420 09:20:03.880247 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x, ips: 10.128.0.28 I0420 09:20:03.880276 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:20:04.758776 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:20:04.758800 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611280-72v6x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:20:23.888307 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0420 09:21:27.771987 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=33 I0420 09:21:58.868557 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0420 09:22:11.896527 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0420 09:22:49.894494 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0420 09:23:30.861190 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0420 09:23:43.777537 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0420 09:23:47.851984 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0420 09:24:25.874081 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0420 09:24:39.878613 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=5 I0420 09:24:59.890478 3808 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0420 09:25:02.858191 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0420 09:25:27.772551 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0420 09:26:01.774751 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0420 09:26:04.776879 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0420 09:26:40.782864 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=32 W0420 09:27:17.416764 3808 duration.go:461] Config duration recorder: measurement expired for pod/openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2db9dl W0420 09:27:22.417531 3808 duration.go:461] Config duration recorder: measurement expired for pod/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafj5x26 I0420 09:27:27.775671 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=28 I0420 09:28:12.892048 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0420 09:28:29.871834 3808 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0420 09:29:45.331376 3808 cni.go:452] [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key ] I0420 09:29:45.408118 3808 cni.go:473] [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions c77d75a690726ae859febaf41ad9f89a0da569fad62c8831f3bcd0c1aa65d481 network default NAD default NAD key default], result "{}", err I0420 09:29:45.463507 3808 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:29:45.463557 3808 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0420 09:29:45.464794 3808 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.27 I0420 09:29:45.464824 3808 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:30:00.200062 3808 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29611290-fpnvr for pod on switch ip-10-0-1-176.ec2.internal I0420 09:30:00.200217 3808 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr I0420 09:30:00.215603 3808 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr] pod update took 15.415102ms I0420 09:30:00.215630 3808 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr] addLogicalPort annotation time took 15.448448ms I0420 09:30:00.216958 3808 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr] addLogicalPort took 16.91291ms, libovsdb time 959.794µs I0420 09:30:00.565527 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key ] I0420 09:30:00.575696 3808 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29611290-fpnvr, hostIfaceName: c2a136e0fc5cb9e, network: default, NAD default, SandboxID: "c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5", PCI device ID: , UID: "d6b07608-e7b6-4f49-9943-0761892f47eb", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0420 09:30:00.625575 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:7c:3f:9c:25:14\",\"name\":\"c2a136e0fc5cb9e\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4769f1f9-cd61-4cb5-9b2c-9a7fb5d921f3\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:30:03.417382 3808 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key ] I0420 09:30:03.497689 3808 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr c2a136e0fc5cb9eb8a2eef9c5730eb314bc2391bf66c484d040a8e46b18043c5 network default NAD default NAD key default], result "{}", err I0420 09:30:03.539858 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0420 09:30:03.539890 3808 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr I0420 09:30:03.540973 3808 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr, ips: 10.128.0.29 I0420 09:30:03.541001 3808 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0420 09:30:04.427293 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:30:04.427318 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611290-fpnvr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:30:04.457073 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:30:04.457122 3808 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:30:04.461586 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 I0420 09:30:04.461608 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29611260-zznn4 I0420 09:30:08.855502 3808 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0420 09:30:45.620276 3808 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:30:45.620321 3808 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0420 09:30:45.627246 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0420 09:30:45.627274 3808 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0420 09:30:45.737694 3808 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-1-176.ec2.internal I0420 09:30:45.737853 3808 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0420 09:30:45.756783 3808 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 18.969163ms I0420 09:30:45.756803 3808 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 18.993898ms I0420 09:30:45.758200 3808 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 20.527111ms, libovsdb time 953.522µs I0420 09:30:46.102133 3808 cni.go:452] [tsf/test-tsf-subscriptions f0a317c5dc08522709ba924116434b405b6bbf7d1bb7313d7f3d04d6f4a5caeb network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions f0a317c5dc08522709ba924116434b405b6bbf7d1bb7313d7f3d04d6f4a5caeb network default NAD default NAD key ] I0420 09:30:46.111810 3808 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: f0a317c5dc08522, network: default, NAD default, SandboxID: "f0a317c5dc08522709ba924116434b405b6bbf7d1bb7313d7f3d04d6f4a5caeb", PCI device ID: , UID: "0743f735-b9c7-474e-bd20-1351c82930a3", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0420 09:30:46.159621 3808 cni.go:473] [tsf/test-tsf-subscriptions f0a317c5dc08522709ba924116434b405b6bbf7d1bb7313d7f3d04d6f4a5caeb network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions f0a317c5dc08522709ba924116434b405b6bbf7d1bb7313d7f3d04d6f4a5caeb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:92:93:c6:24:93\",\"name\":\"f0a317c5dc08522\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b615667-6b1b-4b2d-805b-5a486fc07199\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0420 09:30:46.781675 3808 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9