+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-49.ec2.internal ++ [[ -n ip-10-0-2-49.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-49.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' I0521 19:41:02.029158645 - disable conntrack on geneve port + echo 'I0521 19:41:02.029158645 - disable conntrack on geneve port' + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0521 19:41:02.069999235 - starting ovnkube-node + echo 'I0521 19:41:02.069999235 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-49.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-49.ec2.internal --init-node ip-10-0-2-49.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-49.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0521 19:41:02.124792 3793 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0521 19:41:02.124870 3793 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-f7f5605795.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0521 19:41:02.127467 3793 kube.go:431] Waiting for certificate I0521 19:41:02.127503 3793 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0521 19:41:02.127550 3793 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0521 19:41:02.128469 3793 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0521 19:41:02.164925 3793 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:41:02.164940 3793 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:41:02.169136 3793 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:41:02.169214 3793 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-25v86" I0521 19:41:02.174046 3793 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-25v86" I0521 19:41:02.174110 3793 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0521 19:41:03.128198 3793 kube.go:438] Certificate found I0521 19:41:03.128745 3793 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0521 19:41:03.129814 3793 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0521 19:41:03.188640 3793 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-22 19:36:02 +0000 UTC" deadline="2026-05-22 12:57:29.6315445 +0000 UTC" I0521 19:41:03.188690 3793 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h16m26.442871054s" I0521 19:41:03.207255 3793 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0521 19:41:03.219552 3793 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0521 19:41:03.219624 3793 factory.go:561] Starting watch factory I0521 19:41:03.219673 3793 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0521 19:41:03.219706 3793 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219709 3793 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219724 3793 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219737 3793 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219743 3793 server.go:101] MetricServer registers OVS metrics I0521 19:41:03.219721 3793 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219756 3793 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219706 3793 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219888 3793 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.219737 3793 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.220131 3793 server.go:105] MetricServer registers OVN DB metrics I0521 19:41:03.220223 3793 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.220480 3793 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.220543 3793 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.225140 3793 controller_manager.go:370] Starting the ovnkube controller I0521 19:41:03.225134 3793 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0521 19:41:03.225157 3793 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-49.ec2.internal I0521 19:41:03.225173 3793 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0521 19:41:03.225240 3793 controller_manager.go:395] NBDB zone sync took: 71.689µs I0521 19:41:03.225252 3793 factory.go:561] Starting watch factory I0521 19:41:03.238971 3793 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.239419 3793 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.239757 3793 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0521 19:41:03.239905 3793 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0521 19:41:03.241195 3793 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.241253 3793 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.245095 3793 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.245295 3793 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0521 19:41:03.245331 3793 server.go:109] MetricServer registers OVN Controller metrics I0521 19:41:03.247544 3793 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0521 19:41:03.250255 3793 server.go:113] MetricServer registers OVN Northd metrics I0521 19:41:03.254788 3793 metrics.go:519] OVN Metrics Server starts to run ... I0521 19:41:03.325451 3793 factory.go:1977] *v1.Node informer cache synced successfully I0521 19:41:03.325481 3793 factory.go:1977] *v1.Service informer cache synced successfully I0521 19:41:03.325485 3793 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0521 19:41:03.325489 3793 factory.go:1977] *v1.Pod informer cache synced successfully I0521 19:41:03.325494 3793 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0521 19:41:03.325498 3793 factory.go:1977] *v1.Namespace informer cache synced successfully I0521 19:41:03.325625 3793 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:41:03.325658 3793 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:41:03.325647 3793 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:41:03.325710 3793 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:41:03.329128 3793 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:41:03.329133 3793 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0521 19:41:03.331041 3793 factory.go:1977] *v1.Namespace informer cache synced successfully I0521 19:41:03.331052 3793 factory.go:1977] *v1.Node informer cache synced successfully I0521 19:41:03.331056 3793 factory.go:1977] *v1.Service informer cache synced successfully I0521 19:41:03.331059 3793 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0521 19:41:03.331058 3793 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0521 19:41:03.331066 3793 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0521 19:41:03.331062 3793 factory.go:1977] *v1.Pod informer cache synced successfully I0521 19:41:03.331133 3793 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0521 19:41:03.331122 3793 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.331153 3793 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.334454 3793 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.336434 3793 factory.go:1977] *v1.EgressIP informer cache synced successfully I0521 19:41:03.336469 3793 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0521 19:41:03.336480 3793 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0521 19:41:03.336488 3793 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.336501 3793 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.339525 3793 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.342182 3793 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0521 19:41:03.342203 3793 factory.go:1977] *v1.EgressIP informer cache synced successfully I0521 19:41:03.342240 3793 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.342251 3793 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.345571 3793 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.347551 3793 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0521 19:41:03.347566 3793 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0521 19:41:03.347631 3793 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.347651 3793 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.350398 3793 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.352862 3793 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0521 19:41:03.352882 3793 factory.go:1977] *v1.EgressService informer cache synced successfully I0521 19:41:03.352940 3793 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.352957 3793 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.355876 3793 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.358417 3793 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0521 19:41:03.358434 3793 factory.go:1977] *v1.EgressService informer cache synced successfully I0521 19:41:03.358629 3793 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0521 19:41:03.358646 3793 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0521 19:41:03.361790 3793 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0521 19:41:03.363774 3793 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0521 19:41:03.363788 3793 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0521 19:41:03.363848 3793 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.363861 3793 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.363900 3793 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.363919 3793 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.367213 3793 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.367216 3793 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0521 19:41:03.369103 3793 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0521 19:41:03.369182 3793 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0521 19:41:03.369192 3793 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0521 19:41:03.369199 3793 factory.go:680] Watch Factory start up complete, took: 149.578079ms I0521 19:41:03.369254 3793 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0521 19:41:03.374810 3793 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0521 19:41:03.374820 3793 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0521 19:41:03.374826 3793 factory.go:680] Watch Factory start up complete, took: 149.573219ms I0521 19:41:03.374833 3793 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-49.ec2.internal" zone I0521 19:41:03.416760 3793 default_node_network_controller.go:710] Initializing the default node network controller I0521 19:41:03.496692 3793 udn_isolation.go:101] Starting UDN host isolation manager I0521 19:41:03.502732 3793 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0521 19:41:03.561205 3793 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0521 19:41:03.561270 3793 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0521 19:41:03.561284 3793 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0521 19:41:03.594756 3793 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0521 19:41:03.594821 3793 default_node_network_controller.go:809] Node ip-10-0-2-49.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0521 19:41:03.600534 3793 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.49"] k8s.ovn.org/zone-name:ip-10-0-2-49.ec2.internal] on node ip-10-0-2-49.ec2.internal I0521 19:41:03.622783 3793 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0521 19:41:03.624985 3793 controller_manager.go:429] Waiting for node in zone sync took: 250.14179ms I0521 19:41:03.629093 3793 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0521 19:41:03.629221 3793 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0521 19:41:03.629343 3793 gateway_init.go:263] Preparing Gateway I0521 19:41:03.629355 3793 gateway_shared_intf.go:1687] Creating new gateway I0521 19:41:03.630782 3793 controller_manager.go:611] Switching to transit router for layer2 networks I0521 19:41:03.630796 3793 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-49.ec2.internal I0521 19:41:03.648248 3793 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0521 19:41:03.655482 3793 controller_manager.go:290] SCTP support detected in OVN I0521 19:41:03.656093 3793 services_controller.go:69] Creating services controller for network=default I0521 19:41:03.656156 3793 udn_enabled_svc.go:80] Waiting for service informer to sync I0521 19:41:03.656219 3793 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0521 19:41:03.656248 3793 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0521 19:41:03.656261 3793 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0521 19:41:03.656268 3793 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0521 19:41:03.656450 3793 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0521 19:41:03.656490 3793 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0521 19:41:03.656529 3793 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.824µs I0521 19:41:03.656547 3793 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 49.628µs I0521 19:41:03.656554 3793 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 64.239µs I0521 19:41:03.656559 3793 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0521 19:41:03.656568 3793 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0521 19:41:03.656577 3793 default_network_controller.go:325] Starting the default network controller I0521 19:41:03.656665 3793 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 19.233µs I0521 19:41:03.656731 3793 udn_enabled_svc.go:95] Performing full resync I0521 19:41:03.657039 3793 udn_enabled_svc.go:99] Waiting for handler to sync I0521 19:41:03.657051 3793 udn_enabled_svc.go:104] Starting worker I0521 19:41:03.666224 3793 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0521 19:41:03.666281 3793 repair.go:33] Syncing exgw routes took 44.187µs I0521 19:41:03.666303 3793 default_network_controller.go:414] Starting all the Watchers... I0521 19:41:03.666522 3793 namespace.go:98] [openshift-monitoring] adding namespace I0521 19:41:03.666523 3793 namespace.go:98] [openshift-backplane-csm] adding namespace I0521 19:41:03.666524 3793 namespace.go:98] [openshift-machine-api] adding namespace I0521 19:41:03.666533 3793 namespace.go:98] [openshift-dns] adding namespace I0521 19:41:03.666549 3793 namespace.go:98] [openshift-backplane-cse] adding namespace I0521 19:41:03.666550 3793 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0521 19:41:03.666551 3793 namespace.go:98] [openshift-network-diagnostics] adding namespace I0521 19:41:03.666549 3793 namespace.go:98] [openshift-console] adding namespace I0521 19:41:03.666561 3793 namespace.go:98] [openshift-ingress-canary] adding namespace I0521 19:41:03.666564 3793 namespace.go:98] [openshift-kube-apiserver] adding namespace I0521 19:41:03.666567 3793 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0521 19:41:03.666568 3793 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0521 19:41:03.666574 3793 namespace.go:98] [openshift-etcd] adding namespace I0521 19:41:03.666577 3793 namespace.go:98] [openshift-console-user-settings] adding namespace I0521 19:41:03.666595 3793 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0521 19:41:03.667064 3793 namespace.go:102] [openshift-monitoring] adding namespace took 532.08µs I0521 19:41:03.667078 3793 namespace.go:98] [openshift-cluster-version] adding namespace I0521 19:41:03.667523 3793 namespace.go:102] [openshift-cluster-version] adding namespace took 438.827µs I0521 19:41:03.667539 3793 namespace.go:98] [openshift-ingress-operator] adding namespace I0521 19:41:03.668029 3793 namespace.go:102] [openshift-ingress-operator] adding namespace took 485.268µs I0521 19:41:03.668041 3793 namespace.go:98] [openshift-host-network] adding namespace I0521 19:41:03.668511 3793 namespace.go:102] [openshift-backplane-csm] adding namespace took 1.975292ms I0521 19:41:03.668531 3793 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0521 19:41:03.668981 3793 namespace.go:102] [openshift-dns] adding namespace took 2.43767ms I0521 19:41:03.668992 3793 namespace.go:98] [openshift-backplane] adding namespace I0521 19:41:03.669389 3793 namespace.go:102] [openshift-machine-api] adding namespace took 2.849674ms I0521 19:41:03.669402 3793 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0521 19:41:03.669877 3793 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 3.317213ms I0521 19:41:03.669885 3793 namespace.go:98] [openshift-kube-scheduler] adding namespace W0521 19:41:03.670273 3793 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-mhhd8: pod openshift-network-diagnostics/network-check-target-mhhd8: no pod IPs found I0521 19:41:03.670308 3793 namespace.go:102] [openshift-backplane-cse] adding namespace took 3.752537ms I0521 19:41:03.670316 3793 namespace.go:98] [openshift-backplane-tam] adding namespace I0521 19:41:03.670834 3793 namespace.go:102] [openshift-network-diagnostics] adding namespace took 4.269455ms I0521 19:41:03.670849 3793 namespace.go:98] [openshift-apiserver] adding namespace I0521 19:41:03.671288 3793 namespace.go:102] [openshift-console] adding namespace took 4.719048ms I0521 19:41:03.671305 3793 namespace.go:98] [openshift] adding namespace I0521 19:41:03.671836 3793 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.268409ms I0521 19:41:03.671847 3793 namespace.go:98] [openshift-network-console] adding namespace I0521 19:41:03.672275 3793 namespace.go:102] [openshift-kube-apiserver] adding namespace took 5.699785ms I0521 19:41:03.672290 3793 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0521 19:41:03.672741 3793 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.164157ms I0521 19:41:03.672760 3793 namespace.go:98] [openshift-image-registry] adding namespace I0521 19:41:03.673185 3793 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.608044ms I0521 19:41:03.673197 3793 namespace.go:98] [default] adding namespace I0521 19:41:03.673651 3793 namespace.go:102] [openshift-etcd] adding namespace took 7.070131ms I0521 19:41:03.673666 3793 namespace.go:98] [kube-node-lease] adding namespace I0521 19:41:03.674102 3793 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.519648ms I0521 19:41:03.674113 3793 namespace.go:98] [openshift-package-operator] adding namespace I0521 19:41:03.674592 3793 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.980879ms I0521 19:41:03.674611 3793 namespace.go:98] [openshift-operators-redhat] adding namespace I0521 19:41:03.675053 3793 namespace.go:102] [openshift-host-network] adding namespace took 7.006884ms I0521 19:41:03.675068 3793 namespace.go:98] [openshift-logging] adding namespace I0521 19:41:03.675505 3793 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.967074ms I0521 19:41:03.675519 3793 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0521 19:41:03.676020 3793 namespace.go:102] [openshift-backplane] adding namespace took 7.02031ms I0521 19:41:03.676043 3793 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0521 19:41:03.676491 3793 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.082416ms I0521 19:41:03.676510 3793 namespace.go:98] [openshift-config] adding namespace I0521 19:41:03.677040 3793 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.14874ms I0521 19:41:03.677059 3793 namespace.go:98] [open-cluster-management-2qegovcgb34vgjgrmsm5lc08sjlrclho] adding namespace I0521 19:41:03.677508 3793 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.185841ms I0521 19:41:03.677539 3793 namespace.go:98] [openshift-apiserver-operator] adding namespace I0521 19:41:03.678116 3793 namespace.go:102] [openshift-apiserver] adding namespace took 7.259967ms I0521 19:41:03.678131 3793 namespace.go:98] [openshift-dns-operator] adding namespace I0521 19:41:03.678688 3793 namespace.go:102] [openshift] adding namespace took 7.375618ms I0521 19:41:03.678706 3793 namespace.go:98] [kube-public] adding namespace I0521 19:41:03.679116 3793 namespace.go:102] [openshift-network-console] adding namespace took 7.262915ms I0521 19:41:03.679130 3793 namespace.go:98] [dedicated-admin] adding namespace I0521 19:41:03.679504 3793 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.206962ms I0521 19:41:03.679522 3793 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0521 19:41:03.680073 3793 namespace.go:102] [openshift-image-registry] adding namespace took 7.304976ms I0521 19:41:03.680093 3793 namespace.go:98] [openshift-network-node-identity] adding namespace I0521 19:41:03.680494 3793 namespace.go:102] [default] adding namespace took 7.291092ms I0521 19:41:03.680512 3793 namespace.go:98] [openshift-multus] adding namespace I0521 19:41:03.680918 3793 namespace.go:102] [kube-node-lease] adding namespace took 7.246533ms I0521 19:41:03.680934 3793 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0521 19:41:03.681308 3793 namespace.go:102] [openshift-package-operator] adding namespace took 7.190412ms I0521 19:41:03.681323 3793 namespace.go:98] [openshift-network-operator] adding namespace I0521 19:41:03.681707 3793 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.089373ms I0521 19:41:03.681721 3793 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0521 19:41:03.682127 3793 namespace.go:102] [openshift-logging] adding namespace took 7.053263ms I0521 19:41:03.682668 3793 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.143915ms I0521 19:41:03.682682 3793 namespace.go:98] [openshift-authentication] adding namespace I0521 19:41:03.683109 3793 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.060999ms I0521 19:41:03.683121 3793 namespace.go:98] [openshift-backplane-ai-agent] adding namespace I0521 19:41:03.683651 3793 namespace.go:102] [openshift-config] adding namespace took 7.135434ms I0521 19:41:03.683667 3793 namespace.go:98] [openshift-service-ca-operator] adding namespace I0521 19:41:03.684261 3793 namespace.go:102] [open-cluster-management-2qegovcgb34vgjgrmsm5lc08sjlrclho] adding namespace took 7.19454ms I0521 19:41:03.684278 3793 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0521 19:41:03.684898 3793 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.349286ms I0521 19:41:03.684921 3793 namespace.go:98] [openshift-must-gather-operator] adding namespace I0521 19:41:03.685410 3793 namespace.go:102] [openshift-dns-operator] adding namespace took 7.27162ms I0521 19:41:03.685429 3793 namespace.go:98] [openshift-customer-monitoring] adding namespace I0521 19:41:03.685842 3793 namespace.go:102] [kube-public] adding namespace took 7.129465ms I0521 19:41:03.685858 3793 namespace.go:98] [openshift-route-controller-manager] adding namespace I0521 19:41:03.686260 3793 namespace.go:102] [dedicated-admin] adding namespace took 7.123722ms I0521 19:41:03.686273 3793 namespace.go:98] [openshift-config-managed] adding namespace I0521 19:41:03.686804 3793 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.276478ms I0521 19:41:03.686815 3793 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace W0521 19:41:03.687772 3793 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-w6hwz: pod openshift-multus/network-metrics-daemon-w6hwz: no pod IPs found I0521 19:41:03.687811 3793 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.713339ms I0521 19:41:03.687822 3793 namespace.go:98] [openshift-authentication-operator] adding namespace I0521 19:41:03.688361 3793 namespace.go:102] [openshift-multus] adding namespace took 7.842647ms I0521 19:41:03.688403 3793 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0521 19:41:03.688928 3793 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.986766ms I0521 19:41:03.688946 3793 namespace.go:98] [kube-system] adding namespace I0521 19:41:03.689525 3793 namespace.go:102] [openshift-network-operator] adding namespace took 8.194859ms I0521 19:41:03.689543 3793 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0521 19:41:03.690082 3793 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.35386ms I0521 19:41:03.690099 3793 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0521 19:41:03.690639 3793 namespace.go:102] [openshift-authentication] adding namespace took 7.949987ms I0521 19:41:03.690658 3793 namespace.go:98] [openshift-backplane-srep] adding namespace I0521 19:41:03.691231 3793 namespace.go:102] [openshift-backplane-ai-agent] adding namespace took 8.103342ms I0521 19:41:03.691250 3793 namespace.go:98] [openshift-node] adding namespace I0521 19:41:03.691777 3793 namespace.go:102] [openshift-service-ca-operator] adding namespace took 8.104917ms I0521 19:41:03.691794 3793 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0521 19:41:03.692210 3793 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.926268ms I0521 19:41:03.692223 3793 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0521 19:41:03.692731 3793 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.804565ms I0521 19:41:03.692744 3793 namespace.go:98] [openshift-backplane-mobb] adding namespace I0521 19:41:03.693180 3793 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.743644ms I0521 19:41:03.693198 3793 namespace.go:98] [openshift-console-operator] adding namespace I0521 19:41:03.693859 3793 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.993371ms I0521 19:41:03.693879 3793 namespace.go:98] [openshift-infra] adding namespace I0521 19:41:03.694384 3793 namespace.go:102] [openshift-config-managed] adding namespace took 8.085168ms I0521 19:41:03.694405 3793 namespace.go:98] [openshift-operators] adding namespace I0521 19:41:03.694999 3793 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 8.177583ms I0521 19:41:03.695020 3793 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0521 19:41:03.695641 3793 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.813135ms I0521 19:41:03.695656 3793 namespace.go:98] [openshift-ingress] adding namespace W0521 19:41:03.696049 3793 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-gg8cw: pod kube-system/global-pull-secret-syncer-gg8cw: no pod IPs found I0521 19:41:03.696106 3793 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.69637ms I0521 19:41:03.696126 3793 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0521 19:41:03.696676 3793 namespace.go:102] [kube-system] adding namespace took 7.723535ms I0521 19:41:03.696692 3793 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0521 19:41:03.697247 3793 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.696895ms I0521 19:41:03.697267 3793 namespace.go:98] [openshift-controller-manager] adding namespace I0521 19:41:03.697799 3793 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.693226ms I0521 19:41:03.697812 3793 namespace.go:98] [openshift-insights] adding namespace I0521 19:41:03.698247 3793 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.582948ms I0521 19:41:03.698756 3793 namespace.go:102] [openshift-node] adding namespace took 7.500179ms I0521 19:41:03.699212 3793 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.4119ms I0521 19:41:03.699229 3793 namespace.go:98] [openshift-backplane-cee] adding namespace I0521 19:41:03.699605 3793 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.374954ms I0521 19:41:03.700080 3793 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.331583ms I0521 19:41:03.700094 3793 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0521 19:41:03.700531 3793 namespace.go:102] [openshift-console-operator] adding namespace took 7.328135ms I0521 19:41:03.700956 3793 namespace.go:102] [openshift-infra] adding namespace took 7.069117ms I0521 19:41:03.700973 3793 namespace.go:98] [openshift-config-operator] adding namespace I0521 19:41:03.701326 3793 namespace.go:102] [openshift-operators] adding namespace took 6.916035ms I0521 19:41:03.701840 3793 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.811886ms I0521 19:41:03.701857 3793 namespace.go:98] [openshift-machine-config-operator] adding namespace I0521 19:41:03.702504 3793 namespace.go:102] [openshift-ingress] adding namespace took 6.840037ms I0521 19:41:03.703021 3793 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.888587ms I0521 19:41:03.703652 3793 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.951877ms I0521 19:41:03.703672 3793 namespace.go:98] [openshift-marketplace] adding namespace I0521 19:41:03.704136 3793 namespace.go:102] [openshift-controller-manager] adding namespace took 6.862274ms I0521 19:41:03.704642 3793 namespace.go:102] [openshift-insights] adding namespace took 6.823609ms I0521 19:41:03.705067 3793 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.83281ms I0521 19:41:03.705602 3793 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.502543ms I0521 19:41:03.705992 3793 namespace.go:102] [openshift-config-operator] adding namespace took 5.01316ms I0521 19:41:03.706645 3793 namespace.go:102] [openshift-machine-config-operator] adding namespace took 4.780962ms I0521 19:41:03.707056 3793 namespace.go:102] [openshift-marketplace] adding namespace took 3.377291ms W0521 19:41:03.707157 3793 master.go:458] Unable to parse local node ip-10-0-2-49.ec2.internal chassis-id annotation. Chassis may be removed during sync I0521 19:41:03.707168 3793 master.go:481] Removing stale chassis with ID/Name: 68e1a6ee-4696-430c-a112-4f690c39e800, hostname: ip-10-0-2-49.ec2.internal I0521 19:41:03.707684 3793 master.go:548] Adding or Updating local node "ip-10-0-2-49.ec2.internal" for network "default" I0521 19:41:03.711410 3793 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-49.ec2.internal for the network default I0521 19:41:03.711746 3793 zone_ic_handler.go:195] Time taken to create transit switch: 320.813µs I0521 19:41:03.712475 3793 default_network_controller.go:823] Node add failed for ip-10-0-2-49.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-49.ec2.internal, error getting gateway config for node ip-10-0-2-49.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-49.ec2.internal", failed to update chassis to local for local node ip-10-0-2-49.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-49.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-49.ec2.internal] E0521 19:41:03.712520 3793 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-49.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-49.ec2.internal, error getting gateway config for node ip-10-0-2-49.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-49.ec2.internal", failed to update chassis to local for local node ip-10-0-2-49.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-49.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-49.ec2.internal] I0521 19:41:03.712577 3793 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0521 19:41:03.712598 3793 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0521 19:41:03.712615 3793 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0521 19:41:03.712597 3793 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-49.ec2.internal", UID:"0958df79-6cbd-4565-a531-1b0922a9d151", APIVersion:"v1", ResourceVersion:"8407", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-49.ec2.internal, error getting gateway config for node ip-10-0-2-49.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-49.ec2.internal", failed to update chassis to local for local node ip-10-0-2-49.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-49.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-49.ec2.internal] I0521 19:41:03.712629 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-49.ec2.internal in network "default" I0521 19:41:03.712679 3793 node_tracker.go:235] Node ip-10-0-2-49.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-49.ec2.internal" I0521 19:41:03.712698 3793 node_tracker.go:169] Node ip-10-0-2-49.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:03.712717 3793 services_controller.go:574] Full service sync requested for network=default I0521 19:41:03.747095 3793 iptables.go:108] Creating table: filter chain: FORWARD I0521 19:41:03.754161 3793 iptables.go:108] Creating table: filter chain: OUTPUT I0521 19:41:03.761448 3793 gateway_shared_intf.go:1816] Gateway Creation Complete I0521 19:41:03.761464 3793 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-49.ec2.internal","mac-address":"0e:a2:00:d2:30:19","ip-addresses":["10.0.2.49/24"],"ip-address":"10.0.2.49/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:68e1a6ee-4696-430c-a112-4f690c39e800 k8s.ovn.org/node-encap-ips:["10.0.2.49"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.49/24"} k8s.ovn.org/zone-name:ip-10-0-2-49.ec2.internal] on node ip-10-0-2-49.ec2.internal I0521 19:41:03.779874 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-49.ec2.internal in network "default" I0521 19:41:03.779932 3793 node_tracker.go:169] Node ip-10-0-2-49.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:03.779947 3793 services_controller.go:574] Full service sync requested for network=default I0521 19:41:03.779973 3793 master.go:548] Adding or Updating local node "ip-10-0-2-49.ec2.internal" for network "default" I0521 19:41:03.784221 3793 default_node_network_controller.go:519] Node connection status = connected W0521 19:41:03.785396 3793 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-w6hwz: pod openshift-multus/network-metrics-daemon-w6hwz: no pod IPs found W0521 19:41:03.785413 3793 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-mhhd8: pod openshift-network-diagnostics/network-check-target-mhhd8: no pod IPs found W0521 19:41:03.785419 3793 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-gg8cw: pod kube-system/global-pull-secret-syncer-gg8cw: no pod IPs found I0521 19:41:03.786332 3793 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-49.ec2.internal for the network default I0521 19:41:03.786500 3793 zone_ic_handler.go:195] Time taken to create transit switch: 156.978µs I0521 19:41:03.812791 3793 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0521 19:41:03.812811 3793 services_controller.go:208] Setting up event handlers for services for network=default I0521 19:41:03.812983 3793 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0521 19:41:03.813026 3793 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0521 19:41:03.813041 3793 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0521 19:41:03.913238 3793 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0521 19:41:03.913259 3793 repair.go:60] Starting repairing loop for services I0521 19:41:03.913343 3793 repair.go:131] Deleted 0 stale service LBs I0521 19:41:03.913374 3793 repair.go:137] Deleted 0 stale Chassis Template Vars I0521 19:41:03.913393 3793 repair.go:62] Finished repairing loop for services: 136.201µs I0521 19:41:03.913418 3793 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0521 19:41:03.913428 3793 services_controller.go:254] Starting workers for network=default I0521 19:41:03.913668 3793 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mhhd8] creating logical port openshift-network-diagnostics_network-check-target-mhhd8 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:03.913730 3793 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gg8cw] creating logical port kube-system_global-pull-secret-syncer-gg8cw for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:03.913832 3793 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-w6hwz] creating logical port openshift-multus_network-metrics-daemon-w6hwz for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:03.913871 3793 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-mhhd8 I0521 19:41:03.913932 3793 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-w6hwz I0521 19:41:03.913938 3793 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-gg8cw I0521 19:41:03.929393 3793 pod.go:62] [openshift-network-diagnostics/network-check-target-mhhd8] pod update took 15.635986ms I0521 19:41:03.929411 3793 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-mhhd8] addLogicalPort annotation time took 15.656151ms I0521 19:41:03.930148 3793 pods.go:271] [openshift-network-diagnostics/network-check-target-mhhd8] addLogicalPort took 16.492548ms, libovsdb time 555.93µs I0521 19:41:03.935228 3793 pod.go:62] [kube-system/global-pull-secret-syncer-gg8cw] pod update took 21.426321ms I0521 19:41:03.935244 3793 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-gg8cw] addLogicalPort annotation time took 21.443595ms I0521 19:41:03.936059 3793 pods.go:271] [kube-system/global-pull-secret-syncer-gg8cw] addLogicalPort took 22.336509ms, libovsdb time 632.539µs I0521 19:41:03.936441 3793 pod.go:62] [openshift-multus/network-metrics-daemon-w6hwz] pod update took 22.541249ms I0521 19:41:03.936454 3793 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-w6hwz] addLogicalPort annotation time took 22.555538ms I0521 19:41:03.937190 3793 pods.go:271] [openshift-multus/network-metrics-daemon-w6hwz] addLogicalPort took 23.361987ms, libovsdb time 540.803µs I0521 19:41:03.937785 3793 admin_network_policy_controller.go:227] Starting controller default-network-controller I0521 19:41:03.937859 3793 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0521 19:41:03.937899 3793 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0521 19:41:03.937925 3793 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0521 19:41:03.938244 3793 repair.go:29] Repairing admin network policies took 290.822µs I0521 19:41:03.938523 3793 repair.go:92] Repairing baseline admin network policies took 232.662µs I0521 19:41:03.938802 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0521 19:41:03.938824 3793 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0521 19:41:03.940478 3793 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0521 19:41:03.940502 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0521 19:41:03.940513 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0521 19:41:03.941983 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0521 19:41:03.942006 3793 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0521 19:41:03.942017 3793 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0521 19:41:03.943915 3793 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0521 19:41:03.943936 3793 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0521 19:41:03.943948 3793 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0521 19:41:03.944876 3793 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0521 19:41:03.944896 3793 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0521 19:41:03.944906 3793 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0521 19:41:03.946844 3793 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0521 19:41:03.946865 3793 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0521 19:41:03.946876 3793 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0521 19:41:03.947909 3793 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0521 19:41:03.947929 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0521 19:41:03.947937 3793 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0521 19:41:03.948481 3793 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0521 19:41:03.948500 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0521 19:41:03.948506 3793 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0521 19:41:03.948805 3793 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0521 19:41:03.948821 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0521 19:41:03.948829 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0521 19:41:03.949356 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0521 19:41:03.949393 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0521 19:41:03.949402 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0521 19:41:03.950751 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0521 19:41:03.950770 3793 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0521 19:41:03.950778 3793 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0521 19:41:03.952624 3793 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0521 19:41:03.952644 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0521 19:41:03.952653 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0521 19:41:03.954121 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0521 19:41:03.954141 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0521 19:41:03.954148 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0521 19:41:03.954487 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0521 19:41:03.954506 3793 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0521 19:41:03.954517 3793 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0521 19:41:03.955768 3793 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0521 19:41:03.955786 3793 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0521 19:41:03.955794 3793 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0521 19:41:03.956100 3793 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0521 19:41:03.956119 3793 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0521 19:41:03.956131 3793 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0521 19:41:03.957795 3793 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0521 19:41:03.957812 3793 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0521 19:41:03.957819 3793 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0521 19:41:03.959488 3793 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0521 19:41:03.959504 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0521 19:41:03.959512 3793 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0521 19:41:03.960204 3793 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0521 19:41:03.960222 3793 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0521 19:41:03.960228 3793 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0521 19:41:03.960558 3793 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0521 19:41:03.960573 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0521 19:41:03.960983 3793 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0521 19:41:03.960993 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0521 19:41:03.961694 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0521 19:41:03.961718 3793 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0521 19:41:03.961730 3793 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0521 19:41:03.962746 3793 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0521 19:41:03.962765 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0521 19:41:03.962773 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0521 19:41:03.963336 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0521 19:41:03.963354 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0521 19:41:03.963390 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0521 19:41:03.964100 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0521 19:41:03.964122 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0521 19:41:03.964135 3793 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0521 19:41:03.964688 3793 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0521 19:41:03.964708 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0521 19:41:03.964715 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0521 19:41:03.965212 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0521 19:41:03.965228 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0521 19:41:03.965236 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0521 19:41:03.965769 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0521 19:41:03.965788 3793 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0521 19:41:03.965796 3793 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0521 19:41:03.966262 3793 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0521 19:41:03.966277 3793 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0521 19:41:03.966284 3793 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0521 19:41:03.967293 3793 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0521 19:41:03.967310 3793 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0521 19:41:03.967321 3793 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0521 19:41:03.967872 3793 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0521 19:41:03.971854 3793 egressfirewall.go:374] Starting EgressFirewall controller I0521 19:41:03.971887 3793 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0521 19:41:03.971887 3793 controller.go:133] Adding controller egress-firewall-controller event handlers I0521 19:41:03.971914 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0521 19:41:03.971927 3793 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0521 19:41:03.971916 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0521 19:41:03.971934 3793 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0521 19:41:03.972198 3793 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0521 19:41:03.972211 3793 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0521 19:41:03.972216 3793 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0521 19:41:03.972225 3793 egressqos.go:193] Setting up event handlers for EgressQoS I0521 19:41:03.972233 3793 egressfirewall.go:723] Syncing node "ip-10-0-2-49.ec2.internal" for egress firewall I0521 19:41:03.972297 3793 egressqos.go:245] Starting EgressQoS Controller I0521 19:41:03.972306 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0521 19:41:03.972311 3793 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0521 19:41:03.972316 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0521 19:41:03.972320 3793 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0521 19:41:03.972325 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0521 19:41:03.972329 3793 shared_informer.go:356] "Caches are synced" controller="egressqos" I0521 19:41:03.972333 3793 egressqos.go:259] Repairing EgressQoSes I0521 19:41:03.972337 3793 egressqos.go:399] Starting repairing loop for egressqos I0521 19:41:03.972420 3793 egressqos.go:401] Finished repairing loop for egressqos: 83.251µs I0521 19:41:03.972442 3793 egressservice_zone.go:132] Setting up event handlers for Egress Services I0521 19:41:03.972478 3793 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-49.ec2.internal I0521 19:41:03.972522 3793 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-49.ec2.internal : 46.714µs I0521 19:41:03.972644 3793 egressservice_zone.go:210] Starting Egress Services Controller I0521 19:41:03.972658 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0521 19:41:03.972665 3793 shared_informer.go:356] "Caches are synced" controller="egressservices" I0521 19:41:03.972674 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0521 19:41:03.972679 3793 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0521 19:41:03.972684 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0521 19:41:03.972695 3793 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0521 19:41:03.972700 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0521 19:41:03.972704 3793 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0521 19:41:03.972709 3793 egressservice_zone.go:228] Repairing Egress Services I0521 19:41:03.972966 3793 master_controller.go:88] Starting Admin Policy Based Route Controller I0521 19:41:03.972977 3793 external_controller.go:278] Starting Admin Policy Based Route Controller I0521 19:41:03.972989 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-49.ec2.internal I0521 19:41:03.972998 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-49.ec2.internal: 11.084µs I0521 19:41:03.973136 3793 default_network_controller.go:585] Completing all the Watchers took 306.823051ms I0521 19:41:03.973148 3793 default_network_controller.go:589] Starting unidling controllers I0521 19:41:03.973158 3793 unidle.go:46] Registering OVN SB ControllerEvent handler I0521 19:41:03.973163 3793 unidle.go:63] Populating Initial ContollerEvent events I0521 19:41:03.973174 3793 unidle.go:79] Setting up event handlers for services I0521 19:41:03.973232 3793 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0521 19:41:04.283690 3793 default_node_network_controller.go:519] Node connection status = connected I0521 19:41:04.784039 3793 default_node_network_controller.go:519] Node connection status = connected I0521 19:41:05.284044 3793 default_node_network_controller.go:519] Node connection status = connected I0521 19:41:05.784225 3793 default_node_network_controller.go:519] Node connection status = connected I0521 19:41:05.979116 3793 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0521 19:41:06.283762 3793 default_node_network_controller.go:519] Node connection status = connected I0521 19:41:06.302453 3793 bridgeconfig.go:537] Gateway is ready I0521 19:41:06.302473 3793 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0521 19:41:06.314608 3793 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.49. Updating OVN encap IP. I0521 19:41:06.320351 3793 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.49 - it is already configured I0521 19:41:06.320392 3793 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.49/24:{} 172.20.0.1/32:{}]. Updating annotations. I0521 19:41:06.320649 3793 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.49/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-49.ec2.internal","mac-address":"0e:a2:00:d2:30:19","ip-addresses":["10.0.2.49/24"],"ip-address":"10.0.2.49/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:68e1a6ee-4696-430c-a112-4f690c39e800 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.49/24"}] on node ip-10-0-2-49.ec2.internal I0521 19:41:06.337430 3793 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.49 I0521 19:41:06.337555 3793 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-49.ec2.internal I0521 19:41:06.338393 3793 master.go:548] Adding or Updating local node "ip-10-0-2-49.ec2.internal" for network "default" I0521 19:41:06.338409 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-49.ec2.internal I0521 19:41:06.338419 3793 egressfirewall.go:723] Syncing node "ip-10-0-2-49.ec2.internal" for egress firewall I0521 19:41:06.338424 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-49.ec2.internal: 19.708µs I0521 19:41:06.338449 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-49.ec2.internal in network "default" I0521 19:41:06.338502 3793 node_tracker.go:169] Node ip-10-0-2-49.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:06.338518 3793 services_controller.go:574] Full service sync requested for network=default I0521 19:41:06.343120 3793 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-49.ec2.internal. Updating no re-route policies I0521 19:41:06.343180 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-w6hwz I0521 19:41:06.343189 3793 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-gg8cw I0521 19:41:06.343198 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-w6hwz I0521 19:41:06.343195 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-7xjzm I0521 19:41:06.343203 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-r82dz I0521 19:41:06.343203 3793 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-gg8cw I0521 19:41:06.343213 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-vmrvm I0521 19:41:06.343209 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-7xjzm I0521 19:41:06.343231 3793 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-w6hwz] creating logical port openshift-multus_network-metrics-daemon-w6hwz for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:06.343235 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qbg5q I0521 19:41:06.343235 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-kjxct I0521 19:41:06.343247 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-2fkjw I0521 19:41:06.343251 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-7xjzm after 0 failed attempt(s) I0521 19:41:06.343249 3793 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-49.ec2.internal I0521 19:41:06.343257 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-49.ec2.internal I0521 19:41:06.343258 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-2fkjw I0521 19:41:06.343221 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-r82dz I0521 19:41:06.343269 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-49.ec2.internal I0521 19:41:06.343269 3793 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-49.ec2.internal I0521 19:41:06.343182 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-pml6j I0521 19:41:06.343279 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-r82dz after 0 failed attempt(s) I0521 19:41:06.343279 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-mhhd8 I0521 19:41:06.343283 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-2fkjw after 0 failed attempt(s) I0521 19:41:06.343245 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qbg5q I0521 19:41:06.343294 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-49.ec2.internal after 0 failed attempt(s) I0521 19:41:06.343298 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-mhhd8 I0521 19:41:06.343300 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qbg5q after 0 failed attempt(s) I0521 19:41:06.343294 3793 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-49.ec2.internal after 0 failed attempt(s) I0521 19:41:06.343233 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-vmrvm I0521 19:41:06.343256 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-2jcgm I0521 19:41:06.343317 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-vmrvm after 0 failed attempt(s) I0521 19:41:06.343320 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-2jcgm I0521 19:41:06.343284 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-pml6j I0521 19:41:06.343332 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-2jcgm after 0 failed attempt(s) I0521 19:41:06.343333 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-pml6j after 0 failed attempt(s) I0521 19:41:06.343218 3793 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-z59n9 I0521 19:41:06.343336 3793 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mhhd8] creating logical port openshift-network-diagnostics_network-check-target-mhhd8 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:06.343345 3793 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gg8cw] creating logical port kube-system_global-pull-secret-syncer-gg8cw for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:06.343349 3793 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-z59n9 I0521 19:41:06.343387 3793 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-z59n9 after 0 failed attempt(s) I0521 19:41:06.343262 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-kjxct I0521 19:41:06.343465 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-kjxct after 0 failed attempt(s) I0521 19:41:06.344074 3793 pods.go:271] [openshift-multus/network-metrics-daemon-w6hwz] addLogicalPort took 847.641µs, libovsdb time 276.583µs I0521 19:41:06.344092 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-w6hwz after 0 failed attempt(s) I0521 19:41:06.344195 3793 pods.go:271] [kube-system/global-pull-secret-syncer-gg8cw] addLogicalPort took 843.354µs, libovsdb time 312.496µs I0521 19:41:06.344209 3793 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-gg8cw after 0 failed attempt(s) I0521 19:41:06.344307 3793 pods.go:271] [openshift-network-diagnostics/network-check-target-mhhd8] addLogicalPort took 980.384µs, libovsdb time 428.822µs I0521 19:41:06.344318 3793 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-mhhd8 after 0 failed attempt(s) I0521 19:41:06.359293 3793 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0521 19:41:06.373319 3793 iptables.go:108] Creating table: mangle chain: OUTPUT I0521 19:41:06.376984 3793 iptables.go:108] Creating table: nat chain: OUTPUT I0521 19:41:06.432594 3793 iptables.go:108] Creating table: nat chain: PREROUTING I0521 19:41:06.499211 3793 iptables.go:108] Creating table: filter chain: FORWARD I0521 19:41:06.517027 3793 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0521 19:41:06.517141 3793 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0521 19:41:06.517184 3793 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0521 19:41:06.517253 3793 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0521 19:41:06.517313 3793 gateway_init.go:325] Gateway and management port readiness took 2.738137658s I0521 19:41:06.517341 3793 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0521 19:41:06.517399 3793 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0521 19:41:06.517412 3793 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0521 19:41:06.517422 3793 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0521 19:41:06.517477 3793 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0521 19:41:06.517559 3793 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 55.218µs I0521 19:41:06.517808 3793 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0521 19:41:06.517854 3793 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 23.56µs I0521 19:41:06.517863 3793 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 55.385µs I0521 19:41:06.517868 3793 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0521 19:41:06.517877 3793 nad_controller.go:379] [node-nad-controller NAD controller]: started I0521 19:41:06.517881 3793 default_node_network_controller.go:899] Starting the default node network controller I0521 19:41:06.517886 3793 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0521 19:41:06.517965 3793 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.178µs I0521 19:41:06.518041 3793 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0521 19:41:06.518049 3793 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0521 19:41:06.518073 3793 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a52fc0 0xc0002d1ab0 0xc0002d1b20 0xc0002d1b90 0xc0002d1c00 0xc0002d1c70 0xc0002d1ce0 0xc000031c70 0xc0002d1d50 0xc0002d1dc0 0xc00087eeb0 0xc0064673b0 0xc0002d1e30 map[0x2ff9080:0xc002708960 0x2ff95c0:0xc002708ba0 0x2ffa2e0:0xc0027088a0 0x2ffa580:0xc002708900 0x2ffad60:0xc0027089c0 0x2ffb000:0xc002708a20 0x2ffb2a0:0xc002708b40 0x2ffba80:0xc002708c00 0x2ffbd20:0xc002708c60 0x3026980:0xc0018826c0 0x3026d20:0xc001882720 0x3027800:0xc000cc60c0 0x3029160:0xc000e8af00 0x30298a0:0xc002708780 0x302a720:0xc000d20240] 0xc0004fa4d0 0} I0521 19:41:06.518123 3793 gateway.go:151] Starting gateway service sync I0521 19:41:06.518570 3793 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0521 19:41:06.520911 3793 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0521 19:41:06.541112 3793 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0521 19:41:06.542373 3793 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0521 19:41:06.543552 3793 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0521 19:41:06.577660 3793 gateway.go:168] Gateway service sync done. Time taken: 59.525562ms I0521 19:41:06.577992 3793 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32182 I0521 19:41:06.578465 3793 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a52fc0 0xc0002d1ab0 0xc0002d1b20 0xc0002d1b90 0xc0002d1c00 0xc0002d1c70 0xc0002d1ce0 0xc000031c70 0xc0002d1d50 0xc0002d1dc0 0xc00087eeb0 0xc0064673b0 0xc0002d1e30 map[0x2ff9080:0xc002708960 0x2ff95c0:0xc002708ba0 0x2ffa2e0:0xc0027088a0 0x2ffa580:0xc002708900 0x2ffad60:0xc0027089c0 0x2ffb000:0xc002708a20 0x2ffb2a0:0xc002708b40 0x2ffba80:0xc002708c00 0x2ffbd20:0xc002708c60 0x3026980:0xc0018826c0 0x3026d20:0xc001882720 0x3027800:0xc000cc60c0 0x3029160:0xc000e8af00 0x30298a0:0xc002708780 0x302a720:0xc000d20240] 0xc0004fa4d0 0} I0521 19:41:06.578942 3793 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a52fc0 0xc0002d1ab0 0xc0002d1b20 0xc0002d1b90 0xc0002d1c00 0xc0002d1c70 0xc0002d1ce0 0xc000031c70 0xc0002d1d50 0xc0002d1dc0 0xc00087eeb0 0xc0064673b0 0xc0002d1e30 map[0x2ff9080:0xc002708960 0x2ff95c0:0xc002708ba0 0x2ffa2e0:0xc0027088a0 0x2ffa580:0xc002708900 0x2ffad60:0xc0027089c0 0x2ffb000:0xc002708a20 0x2ffb2a0:0xc002708b40 0x2ffba80:0xc002708c00 0x2ffbd20:0xc002708c60 0x3026980:0xc0018826c0 0x3026d20:0xc001882720 0x3027800:0xc000cc60c0 0x3029160:0xc000e8af00 0x30298a0:0xc002708780 0x302a720:0xc000d20240] 0xc0004fa4d0 0} I0521 19:41:06.578979 3793 gateway.go:532] Reconciling gateway with updates I0521 19:41:06.584112 3793 gateway.go:339] Spawning Conntrack Rule Check Thread I0521 19:41:06.584193 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:41:06.584207 3793 default_node_network_controller.go:944] Gateway and management port readiness took 66.147574ms I0521 19:41:06.584209 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:41:06.584220 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:41:06.584227 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:41:06.584229 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0521 19:41:06.584240 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:41:06.584241 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0521 19:41:06.584210 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:41:06.584253 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:41:06.584251 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0521 19:41:06.584254 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:41:06.584258 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:41:06.584270 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:41:06.584271 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:41:06.584287 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:41:06.584284 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:41:06.584196 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0521 19:41:06.584294 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:41:06.584303 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0521 19:41:06.584304 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:41:06.584307 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0521 19:41:06.584309 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:41:06.584312 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0521 19:41:06.584318 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0521 19:41:06.584323 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0521 19:41:06.584296 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0521 19:41:06.584335 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0521 19:41:06.584337 3793 port_claim.go:71] Svc openshift-ingress/router-default: port 30715 is already open, no action needed I0521 19:41:06.584287 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0521 19:41:06.584342 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0521 19:41:06.584347 3793 port_claim.go:71] Svc openshift-ingress/router-default: port 31879 is already open, no action needed I0521 19:41:06.584378 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0521 19:41:06.584381 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0521 19:41:06.584388 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:41:06.584397 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0521 19:41:06.584401 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:41:06.584397 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:41:06.584404 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:41:06.584410 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:41:06.584412 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:41:06.584413 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:41:06.584412 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:41:06.584423 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:41:06.584425 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:41:06.584427 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:41:06.584429 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0521 19:41:06.584434 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0521 19:41:06.584435 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0521 19:41:06.584419 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:41:06.584448 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0521 19:41:06.584445 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:41:06.584450 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:41:06.584458 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:41:06.584459 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:41:06.584399 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:41:06.584456 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0521 19:41:06.584466 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:41:06.584473 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:41:06.584480 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:41:06.584475 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:41:06.584486 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0521 19:41:06.584483 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:41:06.584494 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0521 19:41:06.584495 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:41:06.584500 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:41:06.584460 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:41:06.584504 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:41:06.584513 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:41:06.584511 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:41:06.584520 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0521 19:41:06.584516 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:41:06.584527 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:41:06.584527 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:41:06.584444 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:41:06.584533 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:41:06.584540 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:41:06.584539 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:41:06.584544 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:41:06.584543 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:41:06.584546 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:41:06.584557 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:41:06.584559 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:41:06.584532 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0521 19:41:06.584584 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:41:06.584591 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0521 19:41:06.584619 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:41:06.584633 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:41:06.584535 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0521 19:41:06.584523 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:41:06.584712 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:41:06.584720 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:41:06.584721 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:41:06.584725 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0521 19:41:06.584728 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:41:06.584733 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:41:06.584735 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0521 19:41:06.584747 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0521 19:41:06.584725 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:41:06.584759 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0521 19:41:06.584714 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:41:06.584769 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:41:06.584713 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:41:06.584858 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:41:06.584872 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:41:06.584845 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:41:06.584712 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:41:06.584962 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:41:06.585242 3793 node_ip_handler_linux.go:160] Node IP manager is running W0521 19:41:06.621296 3793 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0521 19:41:06.621564 3793 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0521 19:41:06.621722 3793 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0521 19:41:06.630305 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0521 19:41:06.630315 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0521 19:41:06.630332 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0521 19:41:06.630338 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0521 19:41:06.630344 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0521 19:41:06.630350 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0521 19:41:06.630355 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0521 19:41:06.630360 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0521 19:41:06.630385 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0521 19:41:06.630391 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0521 19:41:06.669977 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0521 19:41:06.670005 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0521 19:41:06.670012 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0521 19:41:06.670019 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0521 19:41:06.670024 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0521 19:41:06.670029 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0521 19:41:06.670034 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0521 19:41:06.708972 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0521 19:41:06.709002 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0521 19:41:06.709009 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0521 19:41:06.709033 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0521 19:41:06.709049 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0521 19:41:06.709068 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0521 19:41:06.709088 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0521 19:41:06.709100 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0521 19:41:06.709116 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0521 19:41:06.709122 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0521 19:41:07.497391 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0521 19:41:07.497410 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 25.53µs I0521 19:41:07.497428 3793 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0521 19:41:07.497436 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0521 19:41:07.497453 3793 node_tracker.go:220] Node ip-10-0-0-221.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0521 19:41:07.497822 3793 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-221.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0521 19:41:07.497842 3793 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-221.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-221.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-221.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-221.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-221.ec2.internal, error parsing annotation for node ip-10-0-0-221.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0521 19:41:07.509038 3793 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-221.ec2.internal annotation: unexpected end of JSON input I0521 19:41:07.509079 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0521 19:41:07.509101 3793 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-221.ec2.internal annotation: unexpected end of JSON input E0521 19:41:07.509133 3793 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-221.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-221.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-221.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-221.ec2.internal" I0521 19:41:07.526464 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0521 19:41:07.548813 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:07.568330 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0521 19:41:07.568349 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 22.288µs I0521 19:41:07.568329 3793 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0521 19:41:07.568426 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:07.671848 3793 default_node_network_controller.go:1380] Starting node controller node sync W0521 19:41:07.671910 3793 egressip_healthcheck.go:76] Health checking using insecure connection I0521 19:41:07.671957 3793 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0521 19:41:07.683531 3793 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.663258ms I0521 19:41:07.683655 3793 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0521 19:41:07.683915 3793 egressservice_node.go:93] Setting up event handlers for Egress Services I0521 19:41:07.684026 3793 egressservice_node.go:183] Starting Egress Services Controller I0521 19:41:07.684041 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0521 19:41:07.684052 3793 shared_informer.go:356] "Caches are synced" controller="egressservices" I0521 19:41:07.684058 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0521 19:41:07.684063 3793 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0521 19:41:07.684068 3793 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0521 19:41:07.684072 3793 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0521 19:41:07.684076 3793 egressservice_node.go:197] Repairing Egress Services I0521 19:41:07.693790 3793 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0521 19:41:07.693801 3793 external_controller.go:278] Starting Admin Policy Based Route Controller I0521 19:41:07.693938 3793 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0521 19:41:07.693953 3793 link_network_manager.go:119] Link manager is running I0521 19:41:07.693967 3793 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0521 19:41:07.694081 3793 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0521 19:41:07.694618 3793 vrf_manager.go:128] VRF manager is running I0521 19:41:07.694839 3793 node_controller_manager.go:437] Removing flows to drop GARP I0521 19:41:07.694850 3793 gateway.go:532] Reconciling gateway with updates I0521 19:41:07.716317 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:41:07.716322 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:41:07.716329 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:41:07.716336 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0521 19:41:07.716341 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0521 19:41:07.716346 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:41:07.716350 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:41:07.716350 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:41:07.716361 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0521 19:41:07.716379 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0521 19:41:07.716387 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0521 19:41:07.716391 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:41:07.716410 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0521 19:41:07.716410 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:41:07.716413 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:41:07.716417 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0521 19:41:07.716421 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0521 19:41:07.716421 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0521 19:41:07.716342 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0521 19:41:07.716412 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0521 19:41:07.716435 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:41:07.716441 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0521 19:41:07.716445 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0521 19:41:07.716453 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0521 19:41:07.716455 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0521 19:41:07.716462 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716468 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:41:07.716471 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716473 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:41:07.716474 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:41:07.716473 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:41:07.716482 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0521 19:41:07.716482 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0521 19:41:07.716473 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0521 19:41:07.716487 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0521 19:41:07.716490 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0521 19:41:07.716424 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716496 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:41:07.716498 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0521 19:41:07.716487 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:41:07.716504 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:41:07.716509 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0521 19:41:07.716517 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:41:07.716518 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0521 19:41:07.716519 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:41:07.716522 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:41:07.716528 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0521 19:41:07.716531 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:41:07.716530 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:41:07.716535 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:41:07.716538 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:41:07.716535 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0521 19:41:07.716536 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:41:07.716544 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0521 19:41:07.716543 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0521 19:41:07.716544 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0521 19:41:07.716552 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0521 19:41:07.716554 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0521 19:41:07.716555 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0521 19:41:07.716548 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0521 19:41:07.716552 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:41:07.716560 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0521 19:41:07.716516 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0521 19:41:07.716570 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0521 19:41:07.716585 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0521 19:41:07.716594 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716513 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0521 19:41:07.716611 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716505 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0521 19:41:07.716618 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0521 19:41:07.716621 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0521 19:41:07.716485 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0521 19:41:07.716627 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:41:07.716630 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:41:07.716633 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0521 19:41:07.716635 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0521 19:41:07.716638 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0521 19:41:07.716641 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0521 19:41:07.716641 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0521 19:41:07.716511 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0521 19:41:07.716524 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0521 19:41:07.716662 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0521 19:41:07.716492 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716665 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0521 19:41:07.716667 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0521 19:41:07.716671 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:41:07.716530 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0521 19:41:07.716679 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:41:07.716682 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:41:07.716685 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0521 19:41:07.716688 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:41:07.716690 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716693 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:41:07.716695 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:41:07.716695 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:41:07.716705 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0521 19:41:07.716703 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0521 19:41:07.716709 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0521 19:41:07.716709 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0521 19:41:07.716712 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0521 19:41:07.716682 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0521 19:41:07.716685 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0521 19:41:07.716783 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0521 19:41:07.716688 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0521 19:41:07.716795 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:41:07.716802 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0521 19:41:07.716535 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:41:07.716834 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0521 19:41:07.716540 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0521 19:41:07.716545 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:41:07.716881 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:41:07.716547 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0521 19:41:07.716507 3793 port_claim.go:71] Svc openshift-ingress/router-default: port 30715 is already open, no action needed I0521 19:41:07.716931 3793 port_claim.go:71] Svc openshift-ingress/router-default: port 31879 is already open, no action needed I0521 19:41:07.716677 3793 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0521 19:41:07.716963 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0521 19:41:07.716526 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0521 19:41:07.716532 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0521 19:41:07.716703 3793 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0521 19:41:07.720666 3793 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0521 19:41:07.720709 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:07.720725 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0521 19:41:07.720738 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 17.067µs I0521 19:41:07.747698 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:07.756377 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0521 19:41:07.756381 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0521 19:41:07.756389 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0521 19:41:07.756400 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0521 19:41:07.756406 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0521 19:41:07.756412 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0521 19:41:07.756417 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0521 19:41:07.756423 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0521 19:41:07.756428 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0521 19:41:07.756435 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0521 19:41:07.794690 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0521 19:41:07.794711 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0521 19:41:07.794726 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0521 19:41:07.794732 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0521 19:41:07.794738 3793 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0521 19:41:07.919640 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:10.028043 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-49.ec2.internal I0521 19:41:10.028080 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-49.ec2.internal: 45.074µs I0521 19:41:10.064162 3793 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-655886fbdf-zzddl] creating logical port openshift-image-registry_image-registry-655886fbdf-zzddl for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.064297 3793 kube.go:256] Updating pod openshift-image-registry/image-registry-655886fbdf-zzddl I0521 19:41:10.087338 3793 pod.go:62] [openshift-image-registry/image-registry-655886fbdf-zzddl] pod update took 23.065028ms I0521 19:41:10.087389 3793 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-655886fbdf-zzddl] addLogicalPort annotation time took 23.103475ms I0521 19:41:10.088779 3793 pods.go:271] [openshift-image-registry/image-registry-655886fbdf-zzddl] addLogicalPort took 24.641607ms, libovsdb time 1.068213ms I0521 19:41:10.106318 3793 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-l5lrh] creating logical port openshift-dns_dns-default-l5lrh for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.106452 3793 kube.go:256] Updating pod openshift-dns/dns-default-l5lrh I0521 19:41:10.112083 3793 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-854pk for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.112185 3793 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk I0521 19:41:10.113722 3793 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-b95ccdbb6-sd7db] creating logical port openshift-ingress_router-default-b95ccdbb6-sd7db for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.113822 3793 kube.go:256] Updating pod openshift-ingress/router-default-b95ccdbb6-sd7db I0521 19:41:10.130568 3793 pod.go:62] [openshift-dns/dns-default-l5lrh] pod update took 24.13986ms I0521 19:41:10.130596 3793 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-l5lrh] addLogicalPort annotation time took 24.171328ms I0521 19:41:10.131871 3793 pods.go:271] [openshift-dns/dns-default-l5lrh] addLogicalPort took 25.574508ms, libovsdb time 922.751µs I0521 19:41:10.132540 3793 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk] pod update took 20.372992ms I0521 19:41:10.132563 3793 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk] addLogicalPort annotation time took 20.400692ms I0521 19:41:10.134106 3793 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk] addLogicalPort took 22.036487ms, libovsdb time 1.182715ms I0521 19:41:10.140909 3793 pod.go:62] [openshift-ingress/router-default-b95ccdbb6-sd7db] pod update took 27.114777ms I0521 19:41:10.140925 3793 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-b95ccdbb6-sd7db] addLogicalPort annotation time took 27.133736ms I0521 19:41:10.141711 3793 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-k6fk2] creating logical port openshift-console-operator_console-operator-66dbbb6468-k6fk2 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.141803 3793 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-k6fk2 I0521 19:41:10.141895 3793 pods.go:271] [openshift-ingress/router-default-b95ccdbb6-sd7db] addLogicalPort took 28.186788ms, libovsdb time 711.011µs I0521 19:41:10.150235 3793 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-dcmlb for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.150308 3793 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-5jk2l for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.150417 3793 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb I0521 19:41:10.150482 3793 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l I0521 19:41:10.150445 3793 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-2zv6q for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.150580 3793 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-scmcx for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.150608 3793 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q I0521 19:41:10.150665 3793 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx I0521 19:41:10.151048 3793 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-4dq44 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.151124 3793 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-992cd for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.151138 3793 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44 I0521 19:41:10.151177 3793 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd I0521 19:41:10.157675 3793 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-82k2v for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.157759 3793 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v I0521 19:41:10.184068 3793 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-k6fk2] pod update took 42.284404ms I0521 19:41:10.184090 3793 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-k6fk2] addLogicalPort annotation time took 42.31021ms I0521 19:41:10.185192 3793 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-k6fk2] addLogicalPort took 43.485297ms, libovsdb time 840.237µs I0521 19:41:10.185268 3793 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5c5bb958d5-n7tsf] creating logical port openshift-ingress_router-default-5c5bb958d5-n7tsf for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.185382 3793 kube.go:256] Updating pod openshift-ingress/router-default-5c5bb958d5-n7tsf I0521 19:41:10.190247 3793 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v] pod update took 32.503908ms I0521 19:41:10.190253 3793 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd] pod update took 39.089445ms I0521 19:41:10.190267 3793 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v] addLogicalPort annotation time took 32.525655ms I0521 19:41:10.190284 3793 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd] addLogicalPort annotation time took 39.108225ms I0521 19:41:10.191407 3793 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd] addLogicalPort took 40.287488ms, libovsdb time 811.029µs I0521 19:41:10.191494 3793 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-dfg44 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.191539 3793 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v] addLogicalPort took 33.87099ms, libovsdb time 564.324µs I0521 19:41:10.191606 3793 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44 I0521 19:41:10.192406 3793 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-g7n46] creating logical port openshift-ingress-canary_ingress-canary-g7n46 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.192478 3793 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-g7n46 I0521 19:41:10.193231 3793 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx] pod update took 42.591521ms I0521 19:41:10.193250 3793 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx] addLogicalPort annotation time took 42.608143ms I0521 19:41:10.194148 3793 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx] addLogicalPort took 43.577108ms, libovsdb time 644.444µs I0521 19:41:10.194205 3793 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29656540-7gz9d for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.194289 3793 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d I0521 19:41:10.211580 3793 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44] pod update took 60.454411ms I0521 19:41:10.211650 3793 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44] addLogicalPort annotation time took 60.542886ms I0521 19:41:10.213679 3793 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44] addLogicalPort took 62.620564ms, libovsdb time 1.238439ms I0521 19:41:10.213778 3793 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-sf7jz] creating logical port openshift-insights_insights-operator-6dbb575d6c-sf7jz for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:10.213910 3793 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-sf7jz I0521 19:41:10.214399 3793 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q] pod update took 63.808395ms I0521 19:41:10.214422 3793 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q] addLogicalPort annotation time took 63.830828ms I0521 19:41:10.216619 3793 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb] pod update took 66.25542ms I0521 19:41:10.216642 3793 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb] addLogicalPort annotation time took 66.28621ms I0521 19:41:10.216676 3793 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q] addLogicalPort took 66.254126ms, libovsdb time 1.673541ms I0521 19:41:10.216865 3793 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l] pod update took 66.412286ms I0521 19:41:10.216891 3793 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l] addLogicalPort annotation time took 66.471191ms I0521 19:41:10.221640 3793 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l] addLogicalPort took 71.401617ms, libovsdb time 4.192998ms I0521 19:41:10.221952 3793 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb] addLogicalPort took 71.734972ms, libovsdb time 4.73383ms I0521 19:41:10.224346 3793 pod.go:62] [openshift-ingress/router-default-5c5bb958d5-n7tsf] pod update took 39.0231ms I0521 19:41:10.224434 3793 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5c5bb958d5-n7tsf] addLogicalPort annotation time took 39.112071ms I0521 19:41:10.229986 3793 pods.go:271] [openshift-ingress/router-default-5c5bb958d5-n7tsf] addLogicalPort took 44.72514ms, libovsdb time 4.955671ms I0521 19:41:10.231740 3793 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44] pod update took 40.171314ms I0521 19:41:10.231765 3793 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44] addLogicalPort annotation time took 40.206349ms I0521 19:41:10.231776 3793 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d] pod update took 37.526832ms I0521 19:41:10.231775 3793 pod.go:62] [openshift-ingress-canary/ingress-canary-g7n46] pod update took 39.320662ms I0521 19:41:10.231789 3793 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d] addLogicalPort annotation time took 37.54133ms I0521 19:41:10.231796 3793 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-g7n46] addLogicalPort annotation time took 39.344118ms I0521 19:41:10.232950 3793 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44] addLogicalPort took 41.474149ms, libovsdb time 781.922µs I0521 19:41:10.233360 3793 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d] addLogicalPort took 39.160687ms, libovsdb time 1.195244ms I0521 19:41:10.233683 3793 pods.go:271] [openshift-ingress-canary/ingress-canary-g7n46] addLogicalPort took 41.281066ms, libovsdb time 1.476243ms I0521 19:41:10.241921 3793 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-sf7jz] pod update took 28.058823ms I0521 19:41:10.241944 3793 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-sf7jz] addLogicalPort annotation time took 28.089664ms I0521 19:41:10.243038 3793 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-sf7jz] addLogicalPort took 29.270392ms, libovsdb time 712.976µs I0521 19:41:10.549534 3793 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-k6fk2 4934eb14281af26fdba0b7725c79198496b6149cb96c9bd73087e2320896a58f network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-k6fk2 4934eb14281af26fdba0b7725c79198496b6149cb96c9bd73087e2320896a58f network default NAD default NAD key ] I0521 19:41:10.572312 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-k6fk2, hostIfaceName: 4934eb14281af26, network: default, NAD default, SandboxID: "4934eb14281af26fdba0b7725c79198496b6149cb96c9bd73087e2320896a58f", PCI device ID: , UID: "39684949-4ddf-4699-a623-dffe46217853", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0521 19:41:10.595287 3793 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l b32ec6e6ce2fc2ef153d6b7c519b8b66342fbdcd5fff1038439232616d20c0c6 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l b32ec6e6ce2fc2ef153d6b7c519b8b66342fbdcd5fff1038439232616d20c0c6 network default NAD default NAD key ] I0521 19:41:10.605349 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-5jk2l, hostIfaceName: b32ec6e6ce2fc2e, network: default, NAD default, SandboxID: "b32ec6e6ce2fc2ef153d6b7c519b8b66342fbdcd5fff1038439232616d20c0c6", PCI device ID: , UID: "6486a53f-2662-40fe-a7d6-2f80591c6f4a", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0521 19:41:10.625632 3793 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-k6fk2 4934eb14281af26fdba0b7725c79198496b6149cb96c9bd73087e2320896a58f network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-k6fk2 4934eb14281af26fdba0b7725c79198496b6149cb96c9bd73087e2320896a58f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:d6:02:e7:c1:60\",\"name\":\"4934eb14281af26\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6fb5f23e-dfa3-4c30-9108-74a5852218a2\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.651450 3793 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44 f38973215e2d2ec2ec53e5329c69714606db16d884d8e9964ed09b9ec108eb55 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44 f38973215e2d2ec2ec53e5329c69714606db16d884d8e9964ed09b9ec108eb55 network default NAD default NAD key ] I0521 19:41:10.656695 3793 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l b32ec6e6ce2fc2ef153d6b7c519b8b66342fbdcd5fff1038439232616d20c0c6 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-5jk2l b32ec6e6ce2fc2ef153d6b7c519b8b66342fbdcd5fff1038439232616d20c0c6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:bb:da:f9:52:de\",\"name\":\"b32ec6e6ce2fc2e\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a784f76c-62a7-4850-97d3-1d27d909ee53\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.662819 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-4dq44, hostIfaceName: f38973215e2d2ec, network: default, NAD default, SandboxID: "f38973215e2d2ec2ec53e5329c69714606db16d884d8e9964ed09b9ec108eb55", PCI device ID: , UID: "6e57e6f1-cbf1-4354-9d6b-e0012f254982", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0521 19:41:10.673773 3793 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-sf7jz e12895177803afac597d7170f00a661eeed6fce27f20e89551155ba810edb8fe network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-sf7jz e12895177803afac597d7170f00a661eeed6fce27f20e89551155ba810edb8fe network default NAD default NAD key ] I0521 19:41:10.684856 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-sf7jz, hostIfaceName: e12895177803afa, network: default, NAD default, SandboxID: "e12895177803afac597d7170f00a661eeed6fce27f20e89551155ba810edb8fe", PCI device ID: , UID: "9850da56-dca0-44e0-a058-11fe123e143a", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0521 19:41:10.694311 3793 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44 76441176248d3efa78f28181aea9bd52b23dc9e0d308b28e9b5ac5575fbeeacd network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44 76441176248d3efa78f28181aea9bd52b23dc9e0d308b28e9b5ac5575fbeeacd network default NAD default NAD key ] I0521 19:41:10.706711 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-dfg44, hostIfaceName: 76441176248d3ef, network: default, NAD default, SandboxID: "76441176248d3efa78f28181aea9bd52b23dc9e0d308b28e9b5ac5575fbeeacd", PCI device ID: , UID: "794fc5e2-b308-4de4-b33f-f2a852293b9b", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0521 19:41:10.721257 3793 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44 f38973215e2d2ec2ec53e5329c69714606db16d884d8e9964ed09b9ec108eb55 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-4dq44 f38973215e2d2ec2ec53e5329c69714606db16d884d8e9964ed09b9ec108eb55 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:a8:23:83:2a:a2\",\"name\":\"f38973215e2d2ec\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a3aadba3-1b01-4df4-bee9-a6f1a4a8f98b\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.737482 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key ] I0521 19:41:10.744562 3793 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-sf7jz e12895177803afac597d7170f00a661eeed6fce27f20e89551155ba810edb8fe network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-sf7jz e12895177803afac597d7170f00a661eeed6fce27f20e89551155ba810edb8fe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:fa:04:c7:f9:b5\",\"name\":\"e12895177803afa\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8b67906-a67e-43af-83fa-e9445475bcca\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.745325 3793 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v f173b68d0f09c7e7b9e91ab70a8e3448048881bfe857ecd054fe98a88aa743f1 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v f173b68d0f09c7e7b9e91ab70a8e3448048881bfe857ecd054fe98a88aa743f1 network default NAD default NAD key ] I0521 19:41:10.748665 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29656540-7gz9d, hostIfaceName: 3a28d68f9e45758, network: default, NAD default, SandboxID: "3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61", PCI device ID: , UID: "4b224a93-b540-4589-bc20-f377a73aa293", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0521 19:41:10.748985 3793 cni.go:452] [openshift-ingress-canary/ingress-canary-g7n46 9cd1ccffcef1060f28b5d9577fa50ccec2b13cc9a1e64769ecf6611c26683afc network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-g7n46 9cd1ccffcef1060f28b5d9577fa50ccec2b13cc9a1e64769ecf6611c26683afc network default NAD default NAD key ] I0521 19:41:10.762631 3793 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44 76441176248d3efa78f28181aea9bd52b23dc9e0d308b28e9b5ac5575fbeeacd network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-dfg44 76441176248d3efa78f28181aea9bd52b23dc9e0d308b28e9b5ac5575fbeeacd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:5d:e9:57:d1:16\",\"name\":\"76441176248d3ef\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8855bb61-a681-4d47-9cca-054960436646\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.762760 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-82k2v, hostIfaceName: f173b68d0f09c7e, network: default, NAD default, SandboxID: "f173b68d0f09c7e7b9e91ab70a8e3448048881bfe857ecd054fe98a88aa743f1", PCI device ID: , UID: "22d91486-ea0d-4305-8f2b-da505839eb87", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0521 19:41:10.764094 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-g7n46, hostIfaceName: 9cd1ccffcef1060, network: default, NAD default, SandboxID: "9cd1ccffcef1060f28b5d9577fa50ccec2b13cc9a1e64769ecf6611c26683afc", PCI device ID: , UID: "02e4bfbb-d68b-413f-b6a5-b84b10e3af66", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0521 19:41:10.803199 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:c4:9e:2d:de:1c\",\"name\":\"3a28d68f9e45758\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ffbac49-a4cd-4df1-88d7-9d31ff0408c5\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.813655 3793 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v f173b68d0f09c7e7b9e91ab70a8e3448048881bfe857ecd054fe98a88aa743f1 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-82k2v f173b68d0f09c7e7b9e91ab70a8e3448048881bfe857ecd054fe98a88aa743f1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:8c:69:7e:49:17\",\"name\":\"f173b68d0f09c7e\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40416893-5454-45ba-ae5d-b4432774389d\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.819717 3793 cni.go:473] [openshift-ingress-canary/ingress-canary-g7n46 9cd1ccffcef1060f28b5d9577fa50ccec2b13cc9a1e64769ecf6611c26683afc network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-g7n46 9cd1ccffcef1060f28b5d9577fa50ccec2b13cc9a1e64769ecf6611c26683afc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:80:94:c5:08:87\",\"name\":\"9cd1ccffcef1060\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a266f7e5-a56f-4c11-a3ca-28116f42eb44\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:10.970952 3793 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd 46aae890fb84fa8bc83b25c515a1a79494dcd7bf9b942c2357ba7b5a2f82ed81 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd 46aae890fb84fa8bc83b25c515a1a79494dcd7bf9b942c2357ba7b5a2f82ed81 network default NAD default NAD key ] I0521 19:41:10.982152 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-992cd, hostIfaceName: 46aae890fb84fa8, network: default, NAD default, SandboxID: "46aae890fb84fa8bc83b25c515a1a79494dcd7bf9b942c2357ba7b5a2f82ed81", PCI device ID: , UID: "522d395f-80ce-4418-a9d4-15b91b94f5f2", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0521 19:41:11.031238 3793 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd 46aae890fb84fa8bc83b25c515a1a79494dcd7bf9b942c2357ba7b5a2f82ed81 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-992cd 46aae890fb84fa8bc83b25c515a1a79494dcd7bf9b942c2357ba7b5a2f82ed81 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:c7:03:6d:16:f8\",\"name\":\"46aae890fb84fa8\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5ac898b6-9ba0-46e9-b088-e2683c5e2e09\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:11.189146 3793 cni.go:452] [kube-system/global-pull-secret-syncer-gg8cw aa823653c83d20a1cefb03958f02b1fdfa491c8c312069df34172d9ee7149fdc network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-gg8cw aa823653c83d20a1cefb03958f02b1fdfa491c8c312069df34172d9ee7149fdc network default NAD default NAD key ] I0521 19:41:11.200531 3793 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-gg8cw, hostIfaceName: aa823653c83d20a, network: default, NAD default, SandboxID: "aa823653c83d20a1cefb03958f02b1fdfa491c8c312069df34172d9ee7149fdc", PCI device ID: , UID: "52bdb640-74cd-496b-8802-e35e3c476364", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0521 19:41:11.250606 3793 cni.go:473] [kube-system/global-pull-secret-syncer-gg8cw aa823653c83d20a1cefb03958f02b1fdfa491c8c312069df34172d9ee7149fdc network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-gg8cw aa823653c83d20a1cefb03958f02b1fdfa491c8c312069df34172d9ee7149fdc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:d3:ba:08:ba:b9\",\"name\":\"aa823653c83d20a\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eee7c2f7-fc4a-4447-9e5c-e322ea81dd8d\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:11.275871 3793 cni.go:452] [openshift-network-diagnostics/network-check-target-mhhd8 60428a8d6401d090983f43c111caa647c4ef03db63f71b555b31b50138a4f433 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-mhhd8 60428a8d6401d090983f43c111caa647c4ef03db63f71b555b31b50138a4f433 network default NAD default NAD key ] I0521 19:41:11.288473 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-mhhd8, hostIfaceName: 60428a8d6401d09, network: default, NAD default, SandboxID: "60428a8d6401d090983f43c111caa647c4ef03db63f71b555b31b50138a4f433", PCI device ID: , UID: "3b3527eb-6776-41bd-881d-7828de93d87e", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0521 19:41:11.340667 3793 cni.go:473] [openshift-network-diagnostics/network-check-target-mhhd8 60428a8d6401d090983f43c111caa647c4ef03db63f71b555b31b50138a4f433 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-mhhd8 60428a8d6401d090983f43c111caa647c4ef03db63f71b555b31b50138a4f433 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:57:fd:39:ab:f8\",\"name\":\"60428a8d6401d09\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/350fd994-942c-4ff2-8fe4-d66814c5060e\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:11.480464 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:17.857330 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:18.323842 3793 egressfirewall.go:723] Syncing node "ip-10-0-1-105.ec2.internal" for egress firewall I0521 19:41:18.323842 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-105.ec2.internal I0521 19:41:18.323879 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-105.ec2.internal: 45.64µs I0521 19:41:18.323897 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-105.ec2.internal in network "default" I0521 19:41:18.323914 3793 node_tracker.go:220] Node ip-10-0-1-105.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0521 19:41:18.323951 3793 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-105.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0521 19:41:18.323977 3793 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-105.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-105.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-105.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-105.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-105.ec2.internal, error parsing annotation for node ip-10-0-1-105.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0521 19:41:18.340969 3793 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-105.ec2.internal annotation: unexpected end of JSON input I0521 19:41:18.341018 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0521 19:41:18.341050 3793 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-105.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0521 19:41:18.357296 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-105.ec2.internal I0521 19:41:18.357319 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-105.ec2.internal: 28.668µs I0521 19:41:18.357320 3793 egressfirewall.go:723] Syncing node "ip-10-0-1-105.ec2.internal" for egress firewall E0521 19:41:18.357329 3793 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-105.ec2.internal annotation: unexpected end of JSON input I0521 19:41:18.357428 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0521 19:41:18.357464 3793 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-105.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0521 19:41:18.437910 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0521 19:41:18.437922 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:18.437936 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 33.834µs I0521 19:41:18.437910 3793 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall E0521 19:41:18.442213 3793 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-105.ec2.internal annotation: unexpected end of JSON input E0521 19:41:18.442240 3793 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-105.ec2.internal annotation: unexpected end of JSON input I0521 19:41:18.442256 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0521 19:41:18.442320 3793 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-105.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-105.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-105.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-105.ec2.internal" I0521 19:41:18.459343 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0521 19:41:18.572557 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:18.572559 3793 egressfirewall.go:723] Syncing node "ip-10-0-1-105.ec2.internal" for egress firewall I0521 19:41:18.572559 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-105.ec2.internal I0521 19:41:18.572629 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-105.ec2.internal: 74.594µs I0521 19:41:18.594724 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:18.765232 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0521 19:41:21.396989 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0521 19:41:21.552248 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0521 19:41:22.482678 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:41:22.642598 3793 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0521 19:41:23.117359 3793 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0521 19:41:23.118307 3793 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 915.03µs I0521 19:41:23.168435 3793 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0521 19:41:23.785436 3793 namespace.go:98] [openshift-service-ca] adding namespace I0521 19:41:23.786283 3793 namespace.go:102] [openshift-service-ca] adding namespace took 825.72µs I0521 19:41:23.803722 3793 namespace.go:142] [openshift-service-ca] updating namespace I0521 19:41:24.207218 3793 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-ll6sj for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:24.207336 3793 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj I0521 19:41:24.221296 3793 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj] pod update took 13.983416ms I0521 19:41:24.221316 3793 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj] addLogicalPort annotation time took 14.010195ms I0521 19:41:24.222490 3793 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj] addLogicalPort took 15.287353ms, libovsdb time 833.453µs I0521 19:41:24.592442 3793 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj 002de4b0648beaf86f15c3f97e06f4de486fd3d1b788d2d5a03b336f9dfa271b network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj 002de4b0648beaf86f15c3f97e06f4de486fd3d1b788d2d5a03b336f9dfa271b network default NAD default NAD key ] I0521 19:41:24.601934 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-ll6sj, hostIfaceName: 002de4b0648beaf, network: default, NAD default, SandboxID: "002de4b0648beaf86f15c3f97e06f4de486fd3d1b788d2d5a03b336f9dfa271b", PCI device ID: , UID: "ffa978a3-16bc-428a-9cab-b67b468332fd", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0521 19:41:24.651822 3793 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj 002de4b0648beaf86f15c3f97e06f4de486fd3d1b788d2d5a03b336f9dfa271b network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-ll6sj 002de4b0648beaf86f15c3f97e06f4de486fd3d1b788d2d5a03b336f9dfa271b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:61:c1:85:78:9c\",\"name\":\"002de4b0648beaf\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/375d0777-f11e-44a8-b2bb-58c97334c471\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:25.469401 3793 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-bcxvb] creating logical port openshift-service-ca_service-ca-d9cb48497-bcxvb for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:25.469510 3793 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-bcxvb I0521 19:41:25.482563 3793 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-bcxvb] pod update took 13.080673ms I0521 19:41:25.482583 3793 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-bcxvb] addLogicalPort annotation time took 13.107678ms I0521 19:41:25.483874 3793 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-bcxvb] addLogicalPort took 14.507579ms, libovsdb time 938.104µs I0521 19:41:25.844865 3793 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-bcxvb f833b460556c00c54c030212b5e8299b385d46d1082bd335f56c567caaaf32fe network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-bcxvb f833b460556c00c54c030212b5e8299b385d46d1082bd335f56c567caaaf32fe network default NAD default NAD key ] I0521 19:41:25.854492 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-bcxvb, hostIfaceName: f833b460556c00c, network: default, NAD default, SandboxID: "f833b460556c00c54c030212b5e8299b385d46d1082bd335f56c567caaaf32fe", PCI device ID: , UID: "e0104a25-917b-4f6c-b193-e1558d1284f3", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0521 19:41:25.902635 3793 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-bcxvb f833b460556c00c54c030212b5e8299b385d46d1082bd335f56c567caaaf32fe network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-bcxvb f833b460556c00c54c030212b5e8299b385d46d1082bd335f56c567caaaf32fe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:82:ee:0a:42:d4\",\"name\":\"f833b460556c00c\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8d58faed-eb79-4035-9b6c-27087e1de6fc\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:28.052382 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:28.594848 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0521 19:41:31.473563 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0521 19:41:31.597407 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0521 19:41:33.712788 3793 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-0-221.ec2.internal I0521 19:41:33.712821 3793 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-0-221.ec2.internal I0521 19:41:33.712788 3793 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-1-105.ec2.internal I0521 19:41:33.712902 3793 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-1-105.ec2.internal I0521 19:41:33.712928 3793 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-0-221.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-221.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-221.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-221.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-221.ec2.internal I0521 19:41:33.712959 3793 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-1-105.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-105.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-105.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-105.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-105.ec2.internal I0521 19:41:36.248251 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-105.ec2.internal I0521 19:41:36.248277 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-105.ec2.internal: 34.148µs I0521 19:41:36.248278 3793 egressfirewall.go:723] Syncing node "ip-10-0-1-105.ec2.internal" for egress firewall I0521 19:41:36.248392 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:36.344569 3793 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-w8psz I0521 19:41:36.344581 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-ktpl8 I0521 19:41:36.344589 3793 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-5l57d I0521 19:41:36.344600 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-jc68d I0521 19:41:36.344603 3793 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-5l57d I0521 19:41:36.344601 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-ktpl8 I0521 19:41:36.344606 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-jc68d I0521 19:41:36.344604 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-ptcpp I0521 19:41:36.344616 3793 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-59cqd I0521 19:41:36.344621 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-ptcpp I0521 19:41:36.344627 3793 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-59cqd I0521 19:41:36.344633 3793 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-5l57d, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-5l57d: suppressed error logged: pod kube-system/global-pull-secret-syncer-5l57d: no pod IPs found I0521 19:41:36.344634 3793 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-jc68d, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-jc68d: suppressed error logged: pod openshift-multus/network-metrics-daemon-jc68d: no pod IPs found I0521 19:41:36.344592 3793 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-w8psz I0521 19:41:36.344640 3793 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-ktpl8, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-ktpl8: suppressed error logged: pod openshift-multus/network-metrics-daemon-ktpl8: no pod IPs found I0521 19:41:36.344655 3793 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-w8psz, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-w8psz: suppressed error logged: pod kube-system/global-pull-secret-syncer-w8psz: no pod IPs found I0521 19:41:36.344659 3793 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-59cqd, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-59cqd: suppressed error logged: pod openshift-network-diagnostics/network-check-target-59cqd: no pod IPs found I0521 19:41:36.344659 3793 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-ptcpp, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-ptcpp: suppressed error logged: pod openshift-network-diagnostics/network-check-target-ptcpp: no pod IPs found I0521 19:41:38.130416 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:42.240783 3793 cni.go:452] [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key ] I0521 19:41:42.250792 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-655886fbdf-zzddl, hostIfaceName: 096f103ebc0d529, network: default, NAD default, SandboxID: "096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32", PCI device ID: , UID: "2205cf80-1df8-4d74-9821-b528d59e9883", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0521 19:41:42.286887 3793 cni.go:452] [openshift-dns/dns-default-l5lrh ccf56160c967ae141ca7c4e7fe88df30b6e05e9ee1e8200b60d6bdd41b4c2633 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-l5lrh ccf56160c967ae141ca7c4e7fe88df30b6e05e9ee1e8200b60d6bdd41b4c2633 network default NAD default NAD key ] I0521 19:41:42.299893 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-l5lrh, hostIfaceName: ccf56160c967ae1, network: default, NAD default, SandboxID: "ccf56160c967ae141ca7c4e7fe88df30b6e05e9ee1e8200b60d6bdd41b4c2633", PCI device ID: , UID: "7f869831-ccf8-4c0d-9f36-3731bbd5b29b", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0521 19:41:42.303280 3793 cni.go:473] [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:d7:91:9c:1f:bf\",\"name\":\"096f103ebc0d529\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a14ff27c-9b2e-413d-b719-fed57d2411f6\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.313935 3793 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk f56fd42d87b1c7db4b5fa3d344fb08c966fb81994c4a762a7e487a2256d90edc network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk f56fd42d87b1c7db4b5fa3d344fb08c966fb81994c4a762a7e487a2256d90edc network default NAD default NAD key ] I0521 19:41:42.323883 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-854pk, hostIfaceName: f56fd42d87b1c7d, network: default, NAD default, SandboxID: "f56fd42d87b1c7db4b5fa3d344fb08c966fb81994c4a762a7e487a2256d90edc", PCI device ID: , UID: "54e35a62-02e0-4541-8460-3bf6ed9411f5", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0521 19:41:42.336157 3793 cni.go:452] [openshift-ingress/router-default-b95ccdbb6-sd7db 0d58123993dfbf76507b38237fc06bd9d5cbf0af8d8922d33879fc4873bcf4e6 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-b95ccdbb6-sd7db 0d58123993dfbf76507b38237fc06bd9d5cbf0af8d8922d33879fc4873bcf4e6 network default NAD default NAD key ] I0521 19:41:42.348873 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-b95ccdbb6-sd7db, hostIfaceName: 0d58123993dfbf7, network: default, NAD default, SandboxID: "0d58123993dfbf76507b38237fc06bd9d5cbf0af8d8922d33879fc4873bcf4e6", PCI device ID: , UID: "c84e4a4e-4269-4cf9-b871-19a1f6e2b659", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0521 19:41:42.353610 3793 cni.go:473] [openshift-dns/dns-default-l5lrh ccf56160c967ae141ca7c4e7fe88df30b6e05e9ee1e8200b60d6bdd41b4c2633 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-l5lrh ccf56160c967ae141ca7c4e7fe88df30b6e05e9ee1e8200b60d6bdd41b4c2633 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:33:41:9f:24:b5\",\"name\":\"ccf56160c967ae1\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d7a5cb7c-54be-493a-b53b-43a6d84ef814\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.378206 3793 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk f56fd42d87b1c7db4b5fa3d344fb08c966fb81994c4a762a7e487a2256d90edc network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-854pk f56fd42d87b1c7db4b5fa3d344fb08c966fb81994c4a762a7e487a2256d90edc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:e3:70:30:d0:3d\",\"name\":\"f56fd42d87b1c7d\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/98d2672f-75e7-4dce-b07b-273279d65afb\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.398962 3793 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb a9d07c18822584003452799302ae8109009a6c21166fb663e868072727360449 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb a9d07c18822584003452799302ae8109009a6c21166fb663e868072727360449 network default NAD default NAD key ] I0521 19:41:42.404330 3793 cni.go:473] [openshift-ingress/router-default-b95ccdbb6-sd7db 0d58123993dfbf76507b38237fc06bd9d5cbf0af8d8922d33879fc4873bcf4e6 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-b95ccdbb6-sd7db 0d58123993dfbf76507b38237fc06bd9d5cbf0af8d8922d33879fc4873bcf4e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:b2:90:1b:b6:63\",\"name\":\"0d58123993dfbf7\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e81799c-0444-4b58-9307-69e088fc90fd\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.409619 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-dcmlb, hostIfaceName: a9d07c188225840, network: default, NAD default, SandboxID: "a9d07c18822584003452799302ae8109009a6c21166fb663e868072727360449", PCI device ID: , UID: "1a25e40f-0611-4599-8272-79f2b53d7694", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0521 19:41:42.419829 3793 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q f48634c7fed704d7d3f8f1fd278f76aebd615d3689344156a5218bed7a1dec74 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q f48634c7fed704d7d3f8f1fd278f76aebd615d3689344156a5218bed7a1dec74 network default NAD default NAD key ] I0521 19:41:42.423196 3793 cni.go:452] [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key ] I0521 19:41:42.423518 3793 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx fbfffbec98b25be6e0e1ff06ee679a69f625f05a76762b38231ff0c9b8237188 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx fbfffbec98b25be6e0e1ff06ee679a69f625f05a76762b38231ff0c9b8237188 network default NAD default NAD key ] I0521 19:41:42.433480 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-2zv6q, hostIfaceName: f48634c7fed704d, network: default, NAD default, SandboxID: "f48634c7fed704d7d3f8f1fd278f76aebd615d3689344156a5218bed7a1dec74", PCI device ID: , UID: "1b631ad8-865d-463d-91ae-87d75c33111f", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0521 19:41:42.441186 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5c5bb958d5-n7tsf, hostIfaceName: c8c5f832bbb607e, network: default, NAD default, SandboxID: "c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a", PCI device ID: , UID: "4647b649-810f-4a44-8e0a-61593f1cc21c", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0521 19:41:42.441194 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-scmcx, hostIfaceName: fbfffbec98b25be, network: default, NAD default, SandboxID: "fbfffbec98b25be6e0e1ff06ee679a69f625f05a76762b38231ff0c9b8237188", PCI device ID: , UID: "91864387-e079-43dc-93d4-0f4f8c755455", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0521 19:41:42.473611 3793 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb a9d07c18822584003452799302ae8109009a6c21166fb663e868072727360449 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-dcmlb a9d07c18822584003452799302ae8109009a6c21166fb663e868072727360449 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:d5:91:15:89:b2\",\"name\":\"a9d07c188225840\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3c5e919e-772d-4833-b8f3-95c082a80e7d\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.494655 3793 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q f48634c7fed704d7d3f8f1fd278f76aebd615d3689344156a5218bed7a1dec74 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-2zv6q f48634c7fed704d7d3f8f1fd278f76aebd615d3689344156a5218bed7a1dec74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:8a:42:5a:d0:f7\",\"name\":\"f48634c7fed704d\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/977b5627-6f0c-4f15-ae8b-b224e840e778\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.511555 3793 cni.go:473] [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:f5:68:be:17:91\",\"name\":\"c8c5f832bbb607e\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/94f5a65a-b48c-4bd7-a5c9-5fddc3c08e0e\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:42.529903 3793 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx fbfffbec98b25be6e0e1ff06ee679a69f625f05a76762b38231ff0c9b8237188 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-scmcx fbfffbec98b25be6e0e1ff06ee679a69f625f05a76762b38231ff0c9b8237188 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:36:26:9f:de:26\",\"name\":\"fbfffbec98b25be\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1c732cf-13f3-4912-963d-ec9a78a67d9a\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:43.076518 3793 cni.go:452] [openshift-multus/network-metrics-daemon-w6hwz 88c767018cee7fd649a33f11942b202148fca9f90f9ae14068fd22737aea2087 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-w6hwz 88c767018cee7fd649a33f11942b202148fca9f90f9ae14068fd22737aea2087 network default NAD default NAD key ] I0521 19:41:43.087681 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-w6hwz, hostIfaceName: 88c767018cee7fd, network: default, NAD default, SandboxID: "88c767018cee7fd649a33f11942b202148fca9f90f9ae14068fd22737aea2087", PCI device ID: , UID: "3e83fb61-1342-4fea-bb73-96dee54fb531", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0521 19:41:43.144120 3793 cni.go:473] [openshift-multus/network-metrics-daemon-w6hwz 88c767018cee7fd649a33f11942b202148fca9f90f9ae14068fd22737aea2087 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-w6hwz 88c767018cee7fd649a33f11942b202148fca9f90f9ae14068fd22737aea2087 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:ef:80:8f:a4:7d\",\"name\":\"88c767018cee7fd\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01854265-49bb-4f6c-b5f7-8c8d0ea6e45c\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:49.061758 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0521 19:41:49.550215 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:49.550232 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:49.550238 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default I0521 19:41:49.569721 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:49.575330 3793 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0521 19:41:49.589401 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0521 19:41:49.597554 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0521 19:41:49.629953 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0521 19:41:49.655270 3793 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-c65f778fd-879jf IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-c65f778fd-879jf: no pod IPs found I0521 19:41:49.655291 3793 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-dsdjx] creating logical port openshift-insights_insights-runtime-extractor-dsdjx for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:49.655413 3793 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-dsdjx W0521 19:41:49.655431 3793 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-655886fbdf-7t8st IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-655886fbdf-7t8st: no pod IPs found I0521 19:41:49.692454 3793 pod.go:62] [openshift-insights/insights-runtime-extractor-dsdjx] pod update took 37.100997ms I0521 19:41:49.692477 3793 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-dsdjx] addLogicalPort annotation time took 37.128342ms I0521 19:41:49.693693 3793 pods.go:271] [openshift-insights/insights-runtime-extractor-dsdjx] addLogicalPort took 38.420418ms, libovsdb time 894.71µs I0521 19:41:49.750564 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:49.784758 3793 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-b2vf7 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:49.784894 3793 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7 I0521 19:41:49.819466 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:49.826278 3793 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-l7ckr] creating logical port openshift-console_downloads-5c54f4944f-l7ckr for pod on switch ip-10-0-2-49.ec2.internal I0521 19:41:49.826381 3793 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-l7ckr I0521 19:41:49.827914 3793 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7] pod update took 43.054121ms I0521 19:41:49.827932 3793 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7] addLogicalPort annotation time took 43.076711ms I0521 19:41:49.829269 3793 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7] addLogicalPort took 44.534498ms, libovsdb time 1.027305ms I0521 19:41:49.862485 3793 pod.go:62] [openshift-console/downloads-5c54f4944f-l7ckr] pod update took 36.145382ms I0521 19:41:49.862510 3793 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-l7ckr] addLogicalPort annotation time took 36.173598ms I0521 19:41:49.863733 3793 pods.go:271] [openshift-console/downloads-5c54f4944f-l7ckr] addLogicalPort took 37.466715ms, libovsdb time 785.493µs I0521 19:41:50.060350 3793 cni.go:452] [openshift-insights/insights-runtime-extractor-dsdjx 4d6ac31e3c77a5f0198e26ec330b3957615947f1a3c261288a93972b65150469 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-dsdjx 4d6ac31e3c77a5f0198e26ec330b3957615947f1a3c261288a93972b65150469 network default NAD default NAD key ] I0521 19:41:50.069544 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-dsdjx, hostIfaceName: 4d6ac31e3c77a5f, network: default, NAD default, SandboxID: "4d6ac31e3c77a5f0198e26ec330b3957615947f1a3c261288a93972b65150469", PCI device ID: , UID: "58337f6d-d069-4e46-a73d-176e9302e087", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0521 19:41:50.120815 3793 cni.go:473] [openshift-insights/insights-runtime-extractor-dsdjx 4d6ac31e3c77a5f0198e26ec330b3957615947f1a3c261288a93972b65150469 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-dsdjx 4d6ac31e3c77a5f0198e26ec330b3957615947f1a3c261288a93972b65150469 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:12:29:23:72:a2\",\"name\":\"4d6ac31e3c77a5f\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d8b1186-fc0c-44e5-8566-7dcc3c4ad4cf\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:50.165398 3793 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7 86f159eb89d0927c09b3dbec4c011d6995a1b76d3b534d0423934f8ea0a8cb5a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7 86f159eb89d0927c09b3dbec4c011d6995a1b76d3b534d0423934f8ea0a8cb5a network default NAD default NAD key ] I0521 19:41:50.175779 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-b2vf7, hostIfaceName: 86f159eb89d0927, network: default, NAD default, SandboxID: "86f159eb89d0927c09b3dbec4c011d6995a1b76d3b534d0423934f8ea0a8cb5a", PCI device ID: , UID: "3c3aaef0-4ade-4b94-8fad-e42e4d1485e2", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0521 19:41:50.217863 3793 cni.go:452] [openshift-console/downloads-5c54f4944f-l7ckr 313b0792bd2e61329d1d154858840f39de79598846da1391e8c79ed2f9ac795b network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-l7ckr 313b0792bd2e61329d1d154858840f39de79598846da1391e8c79ed2f9ac795b network default NAD default NAD key ] I0521 19:41:50.228180 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-l7ckr, hostIfaceName: 313b0792bd2e613, network: default, NAD default, SandboxID: "313b0792bd2e61329d1d154858840f39de79598846da1391e8c79ed2f9ac795b", PCI device ID: , UID: "2368d156-29d9-41a0-88b4-943260c4148d", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0521 19:41:50.232234 3793 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7 86f159eb89d0927c09b3dbec4c011d6995a1b76d3b534d0423934f8ea0a8cb5a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-b2vf7 86f159eb89d0927c09b3dbec4c011d6995a1b76d3b534d0423934f8ea0a8cb5a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:da:ac:e4:d3:02\",\"name\":\"86f159eb89d0927\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3624792-3a94-4137-838a-3e00dd734486\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:50.283361 3793 cni.go:473] [openshift-console/downloads-5c54f4944f-l7ckr 313b0792bd2e61329d1d154858840f39de79598846da1391e8c79ed2f9ac795b network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-l7ckr 313b0792bd2e61329d1d154858840f39de79598846da1391e8c79ed2f9ac795b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:1d:ca:45:54:40\",\"name\":\"313b0792bd2e613\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79084b7e-f368-480d-88b7-4b6d8a5cd2bb\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:41:50.559289 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-105.ec2.internal in network "default" I0521 19:41:50.559338 3793 node_tracker.go:235] Node ip-10-0-1-105.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-105.ec2.internal" I0521 19:41:50.559346 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "ip-10-0-1-105.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:50.559354 3793 node_tracker.go:169] Node ip-10-0-1-105.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:50.559381 3793 services_controller.go:574] Full service sync requested for network=default W0521 19:41:50.559615 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.559630 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.559635 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.559758 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:50.559822 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0521 19:41:50.581598 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0521 19:41:50.581628 3793 node_tracker.go:235] Node ip-10-0-0-221.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-221.ec2.internal" I0521 19:41:50.581643 3793 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:50.581651 3793 services_controller.go:574] Full service sync requested for network=default I0521 19:41:50.581651 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "ip-10-0-0-221.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0521 19:41:50.581764 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.581776 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.581780 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.581935 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:50.582001 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0521 19:41:50.636625 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "ip-10-0-1-105.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:50.681831 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-105.ec2.internal in network "default" I0521 19:41:50.681898 3793 node_tracker.go:169] Node ip-10-0-1-105.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:50.681905 3793 default_network_controller.go:1025] Node "ip-10-0-1-105.ec2.internal" in remote zone "ip-10-0-1-105.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:50.681916 3793 services_controller.go:574] Full service sync requested for network=default W0521 19:41:50.682149 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.682164 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.682169 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.682298 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:50.682349 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0521 19:41:50.683025 3793 zone_ic_handler.go:195] Time taken to create transit switch: 334.52µs I0521 19:41:50.683039 3793 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-105.ec2.internal for the network default I0521 19:41:50.684149 3793 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-105.ec2.internal" on network "default" took: 1.475847ms I0521 19:41:50.784678 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0521 19:41:50.784685 3793 default_network_controller.go:1025] Node "ip-10-0-0-221.ec2.internal" in remote zone "ip-10-0-0-221.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0521 19:41:50.784741 3793 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:50.784753 3793 services_controller.go:574] Full service sync requested for network=default W0521 19:41:50.784853 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.784864 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.784869 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:50.785126 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:50.785184 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0521 19:41:50.785717 3793 zone_ic_handler.go:195] Time taken to create transit switch: 298.823µs I0521 19:41:50.785737 3793 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-221.ec2.internal for the network default I0521 19:41:50.786732 3793 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-221.ec2.internal" on network "default" took: 1.331267ms I0521 19:41:51.235817 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-105.ec2.internal I0521 19:41:51.235843 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-105.ec2.internal: 34.857µs I0521 19:41:51.235844 3793 egressfirewall.go:723] Syncing node "ip-10-0-1-105.ec2.internal" for egress firewall I0521 19:41:51.235882 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-105.ec2.internal in network "default" I0521 19:41:51.235940 3793 node_tracker.go:169] Node ip-10-0-1-105.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:51.235952 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-105.ec2.internal", IP: 10.0.1.105 to PMTUD blocking rules I0521 19:41:51.235957 3793 services_controller.go:574] Full service sync requested for network=default I0521 19:41:51.235967 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-105.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0521 19:41:51.236056 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:41:51.236317 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:51.236328 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:51.236335 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:51.236468 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0521 19:41:51.242757 3793 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-105.ec2.internal. Updating no re-route policies I0521 19:41:51.251517 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-105.ec2.internal", IP: 10.0.1.105 to PMTUD blocking rules I0521 19:41:51.251531 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-105.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0521 19:41:51.344033 3793 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-221.ec2.internal in network "default" I0521 19:41:51.344074 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 10.0.0.221 to PMTUD blocking rules I0521 19:41:51.344081 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0521 19:41:51.344087 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0521 19:41:51.344096 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 19.514µs I0521 19:41:51.344100 3793 node_tracker.go:169] Node ip-10-0-0-221.ec2.internal switch + router changed, syncing services in network "default" I0521 19:41:51.344106 3793 egressfirewall.go:723] Syncing node "ip-10-0-0-221.ec2.internal" for egress firewall I0521 19:41:51.344115 3793 services_controller.go:574] Full service sync requested for network=default W0521 19:41:51.344267 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:51.344358 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0521 19:41:51.344594 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:51.344609 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:51.344619 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default I0521 19:41:51.350385 3793 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-221.ec2.internal. Updating no re-route policies I0521 19:41:51.370122 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 10.0.0.221 to PMTUD blocking rules I0521 19:41:51.370142 3793 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-221.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0521 19:41:52.577477 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0521 19:41:52.607425 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0521 19:41:53.427814 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0521 19:41:54.357726 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-221.ec2.internal I0521 19:41:54.357757 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-221.ec2.internal: 37.688µs W0521 19:41:54.465646 3793 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:54.465672 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:54.465679 3793 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0521 19:41:55.086689 3793 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-105.ec2.internal I0521 19:41:55.086722 3793 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-105.ec2.internal: 41.959µs W0521 19:41:55.619292 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:56.429076 3793 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0521 19:41:57.557498 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:57.557518 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0521 19:41:57.557524 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0521 19:42:01.474196 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0521 19:42:01.665184 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0521 19:42:03.526220 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0521 19:42:03.890345 3793 base_network_controller_pods.go:487] [default/openshift-console/console-759f8f85-fq4hv] creating logical port openshift-console_console-759f8f85-fq4hv for pod on switch ip-10-0-2-49.ec2.internal I0521 19:42:03.890491 3793 kube.go:256] Updating pod openshift-console/console-759f8f85-fq4hv I0521 19:42:03.913936 3793 pod.go:62] [openshift-console/console-759f8f85-fq4hv] pod update took 23.47975ms I0521 19:42:03.913962 3793 base_network_controller_pods.go:951] [default/openshift-console/console-759f8f85-fq4hv] addLogicalPort annotation time took 23.509044ms I0521 19:42:03.915255 3793 pods.go:271] [openshift-console/console-759f8f85-fq4hv] addLogicalPort took 24.928682ms, libovsdb time 919.518µs W0521 19:42:03.943923 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0521 19:42:05.353384 3793 cni.go:452] [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key ] I0521 19:42:05.362783 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-759f8f85-fq4hv, hostIfaceName: 61898a397055054, network: default, NAD default, SandboxID: "61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f", PCI device ID: , UID: "63fe01bd-e4b4-448d-9c95-a7c1f1e16e22", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0521 19:42:05.414806 3793 cni.go:473] [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:39:f0:01:75:1e\",\"name\":\"61898a397055054\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ec7baf2-1bd4-4a13-9fc6-93fecdd7fe3d\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:42:05.532685 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:42:05.532702 3793 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:42:05.532709 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:42:05.638991 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0521 19:42:06.682922 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0521 19:42:09.520712 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:42:09.520737 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:42:10.537818 3793 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-759f8f85-2gfwv IP address from the namespace address-set, err: pod openshift-console/console-759f8f85-2gfwv: no pod IPs found W0521 19:42:11.572103 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0521 19:42:11.611153 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:11.611180 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:11.611188 3793 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:11.611192 3793 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:11.720219 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:42:11.720238 3793 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:42:11.720246 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0521 19:42:11.762493 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:42:11.762516 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0521 19:42:12.778887 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0521 19:42:14.229280 3793 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0521 19:42:16.845168 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:16.845184 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:16.845189 3793 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0521 19:42:16.845193 3793 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0521 19:42:29.245542 3793 obj_retry.go:498] Detected object openshift-console/console-77fd6dc794-wlbcx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:29.246084 3793 obj_retry.go:498] Detected object openshift-console/console-77fd6dc794-wlbcx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:29.591598 3793 obj_retry.go:492] Detected object openshift-console/console-77fd6dc794-wlbcx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:29.591618 3793 obj_retry.go:492] Detected object openshift-console/console-77fd6dc794-wlbcx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:29.599950 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77fd6dc794-wlbcx I0521 19:42:29.599973 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77fd6dc794-wlbcx I0521 19:42:35.832677 3793 cni.go:452] [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key ] I0521 19:42:35.898733 3793 cni.go:473] [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-759f8f85-fq4hv 61898a3970550545fdf324f5f4deac38dd61e2558940d214b8b7b0db7bd7726f network default NAD default NAD key default], result "{}", err I0521 19:42:35.963403 3793 obj_retry.go:498] Detected object openshift-console/console-759f8f85-fq4hv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:35.963439 3793 pods.go:176] Deleting pod: openshift-console/console-759f8f85-fq4hv I0521 19:42:35.964656 3793 pods.go:236] Attempting to release IPs for pod: openshift-console/console-759f8f85-fq4hv, ips: 10.128.0.28 I0521 19:42:35.964687 3793 obj_retry.go:498] Detected object openshift-console/console-759f8f85-fq4hv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:35.971328 3793 base_network_controller_pods.go:487] [default/openshift-console/console-6d7bf7f474-qkh2w] creating logical port openshift-console_console-6d7bf7f474-qkh2w for pod on switch ip-10-0-2-49.ec2.internal I0521 19:42:35.971445 3793 kube.go:256] Updating pod openshift-console/console-6d7bf7f474-qkh2w I0521 19:42:35.986856 3793 pod.go:62] [openshift-console/console-6d7bf7f474-qkh2w] pod update took 15.437362ms I0521 19:42:35.986883 3793 base_network_controller_pods.go:951] [default/openshift-console/console-6d7bf7f474-qkh2w] addLogicalPort annotation time took 15.466787ms I0521 19:42:35.988298 3793 pods.go:271] [openshift-console/console-6d7bf7f474-qkh2w] addLogicalPort took 16.988025ms, libovsdb time 899.273µs I0521 19:42:36.374163 3793 cni.go:452] [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key ] I0521 19:42:36.383865 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6d7bf7f474-qkh2w, hostIfaceName: 9ac894bc168cee6, network: default, NAD default, SandboxID: "9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3", PCI device ID: , UID: "de5936de-53b2-4aa7-827b-c0fa3de6ed3c", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0521 19:42:36.435358 3793 cni.go:473] [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:10:54:4c:42:61\",\"name\":\"9ac894bc168cee6\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f1f688e0-e762-4e44-bfad-56d1e7911c92\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:42:36.794431 3793 obj_retry.go:492] Detected object openshift-console/console-759f8f85-fq4hv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:36.794457 3793 obj_retry.go:492] Detected object openshift-console/console-759f8f85-fq4hv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:36.798532 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-759f8f85-fq4hv I0521 19:42:36.798578 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-759f8f85-fq4hv W0521 19:42:39.649589 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:42:39.649618 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:42:39.698152 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:42:39.698177 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0521 19:42:41.874950 3793 cni.go:452] [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key ] I0521 19:42:41.951196 3793 cni.go:473] [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-655886fbdf-zzddl 096f103ebc0d52943f4d13327ef5644ad6027a653d997333be5790b8c13c3f32 network default NAD default NAD key default], result "{}", err I0521 19:42:42.026333 3793 obj_retry.go:498] Detected object openshift-image-registry/image-registry-655886fbdf-zzddl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:42.026388 3793 pods.go:176] Deleting pod: openshift-image-registry/image-registry-655886fbdf-zzddl I0521 19:42:42.027688 3793 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-655886fbdf-zzddl, ips: 10.128.0.6 I0521 19:42:42.027713 3793 obj_retry.go:498] Detected object openshift-image-registry/image-registry-655886fbdf-zzddl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0521 19:42:42.815912 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0521 19:42:42.815921 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0521 19:42:42.815967 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0521 19:42:42.826608 3793 obj_retry.go:492] Detected object openshift-image-registry/image-registry-655886fbdf-zzddl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:42.826632 3793 obj_retry.go:492] Detected object openshift-image-registry/image-registry-655886fbdf-zzddl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:42.837460 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-655886fbdf-zzddl I0521 19:42:42.837496 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-655886fbdf-zzddl W0521 19:42:43.819604 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0521 19:42:46.841282 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0521 19:42:52.862669 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0521 19:42:52.862761 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0521 19:42:52.862806 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0521 19:42:52.919790 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0521 19:42:52.919848 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0521 19:42:52.919793 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0521 19:42:53.872129 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key ] W0521 19:42:53.876129 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0521 19:42:53.908142 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0521 19:42:53.957294 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d 3a28d68f9e45758c993a7bb012182e06d73477f3aee9465dc553300725072d61 network default NAD default NAD key default], result "{}", err I0521 19:42:54.016863 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:54.016901 3793 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d I0521 19:42:54.018237 3793 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d, ips: 10.128.0.21 I0521 19:42:54.018270 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:42:54.922243 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:42:54.922271 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:00.227554 3793 cni.go:452] [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key ] I0521 19:43:00.304927 3793 cni.go:473] [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-5c5bb958d5-n7tsf c8c5f832bbb607e115f84565d39fcdc104dcd5910c38d1d73c977d470ccef60a network default NAD default NAD key default], result "{}", err I0521 19:43:00.355067 3793 obj_retry.go:498] Detected object openshift-ingress/router-default-5c5bb958d5-n7tsf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:00.355096 3793 pods.go:176] Deleting pod: openshift-ingress/router-default-5c5bb958d5-n7tsf I0521 19:43:00.356266 3793 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5c5bb958d5-n7tsf, ips: 10.128.0.18 I0521 19:43:00.356295 3793 obj_retry.go:498] Detected object openshift-ingress/router-default-5c5bb958d5-n7tsf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:00.893878 3793 obj_retry.go:492] Detected object openshift-ingress/router-default-5c5bb958d5-n7tsf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:00.893899 3793 obj_retry.go:492] Detected object openshift-ingress/router-default-5c5bb958d5-n7tsf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:00.901317 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5c5bb958d5-n7tsf I0521 19:43:00.901347 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5c5bb958d5-n7tsf I0521 19:43:05.016446 3793 obj_retry.go:498] Detected object openshift-console/console-77fd6dc794-ps4rr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:05.016973 3793 obj_retry.go:498] Detected object openshift-console/console-77fd6dc794-ps4rr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:05.963614 3793 obj_retry.go:492] Detected object openshift-console/console-77fd6dc794-ps4rr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:05.963635 3793 obj_retry.go:492] Detected object openshift-console/console-77fd6dc794-ps4rr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:05.970682 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-77fd6dc794-ps4rr I0521 19:43:05.970702 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-77fd6dc794-ps4rr I0521 19:43:23.578971 3793 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:23.579528 3793 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:24.045448 3793 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:24.045470 3793 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:24.055361 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0521 19:43:24.055411 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0521 19:43:27.943229 3793 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:27.943804 3793 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:28.073650 3793 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:28.073677 3793 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:28.081864 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0521 19:43:28.081892 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0521 19:43:31.973581 3793 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz] creating logical port openshift-deployment-validation-operator_99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz for pod on switch ip-10-0-2-49.ec2.internal I0521 19:43:31.973673 3793 kube.go:256] Updating pod openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz I0521 19:43:31.987858 3793 pod.go:62] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz] pod update took 14.212037ms I0521 19:43:31.987882 3793 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz] addLogicalPort annotation time took 14.240478ms I0521 19:43:31.989109 3793 pods.go:271] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz] addLogicalPort took 15.543534ms, libovsdb time 881.965µs I0521 19:43:32.330390 3793 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key ] I0521 19:43:32.340390 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: 99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz, hostIfaceName: 14bfb4773f89c74, network: default, NAD default, SandboxID: "14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c", PCI device ID: , UID: "4c40f3f7-7b0e-4f91-b105-b31af6e11e1a", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0521 19:43:32.391544 3793 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:a7:d7:a8:7c:5a\",\"name\":\"14bfb4773f89c74\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5bd8e1e2-35ae-4594-86a5-fd37e0048b3a\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:43:48.102717 3793 cni.go:452] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key ] DEL starting CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key ] I0521 19:43:48.167528 3793 cni.go:473] [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key default] DEL finished CNI request [openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz 14bfb4773f89c74e6cdf3b1fd663317b4c8df2f252fba406dc2ebe7080832a3c network default NAD default NAD key default], result "{}", err I0521 19:43:48.214280 3793 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:48.214319 3793 pods.go:176] Deleting pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz I0521 19:43:48.215693 3793 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz, ips: 10.128.0.30 I0521 19:43:48.215721 3793 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:48.215743 3793 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:43:48.215755 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0521 19:43:48.215922 3793 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:43:48.215935 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0521 19:43:49.241817 3793 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:49.241838 3793 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:49.241852 3793 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:49.241863 3793 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d9s5gz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:52.061226 3793 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0521 19:43:52.072241 3793 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0521 19:43:55.811827 3793 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:55.812395 3793 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:43:55.935611 3793 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:55.935630 3793 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:43:55.943072 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0521 19:43:55.943098 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0521 19:44:09.312335 3793 obj_retry.go:498] Detected object openshift-console/console-6d7bf7f474-dvfsr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:44:09.312909 3793 obj_retry.go:498] Detected object openshift-console/console-6d7bf7f474-dvfsr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:44:10.016555 3793 obj_retry.go:492] Detected object openshift-console/console-6d7bf7f474-dvfsr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:44:10.016579 3793 obj_retry.go:492] Detected object openshift-console/console-6d7bf7f474-dvfsr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:44:10.023334 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6d7bf7f474-dvfsr I0521 19:44:10.023360 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6d7bf7f474-dvfsr I0521 19:44:16.831739 3793 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0521 19:44:20.409711 3793 cni.go:452] [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key ] I0521 19:44:20.492492 3793 cni.go:473] [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6d7bf7f474-qkh2w 9ac894bc168cee6c24981f8fb7a3f1f760a26869d21da32952283790543479e3 network default NAD default NAD key default], result "{}", err I0521 19:44:20.557525 3793 obj_retry.go:498] Detected object openshift-console/console-6d7bf7f474-qkh2w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:44:20.557578 3793 pods.go:176] Deleting pod: openshift-console/console-6d7bf7f474-qkh2w I0521 19:44:20.558908 3793 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6d7bf7f474-qkh2w, ips: 10.128.0.29 I0521 19:44:20.558937 3793 obj_retry.go:498] Detected object openshift-console/console-6d7bf7f474-qkh2w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:44:21.235132 3793 obj_retry.go:492] Detected object openshift-console/console-6d7bf7f474-qkh2w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:44:21.235162 3793 obj_retry.go:492] Detected object openshift-console/console-6d7bf7f474-qkh2w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:44:21.242031 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6d7bf7f474-qkh2w I0521 19:44:21.242050 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6d7bf7f474-qkh2w I0521 19:44:30.654802 3793 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:44:30.655354 3793 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:44:31.083600 3793 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:44:31.083625 3793 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:44:31.091025 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0521 19:44:31.091055 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0521 19:45:47.841442 3793 base_network_controller_pods.go:487] [default/openshift-console/console-76df66f86b-blj9b] creating logical port openshift-console_console-76df66f86b-blj9b for pod on switch ip-10-0-2-49.ec2.internal I0521 19:45:47.841584 3793 kube.go:256] Updating pod openshift-console/console-76df66f86b-blj9b I0521 19:45:47.878697 3793 pod.go:62] [openshift-console/console-76df66f86b-blj9b] pod update took 37.144139ms I0521 19:45:47.878721 3793 base_network_controller_pods.go:951] [default/openshift-console/console-76df66f86b-blj9b] addLogicalPort annotation time took 37.173112ms I0521 19:45:47.879939 3793 pods.go:271] [openshift-console/console-76df66f86b-blj9b] addLogicalPort took 38.514851ms, libovsdb time 872.841µs I0521 19:45:48.192172 3793 cni.go:452] [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key ] I0521 19:45:48.201820 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-76df66f86b-blj9b, hostIfaceName: d2165572207988b, network: default, NAD default, SandboxID: "d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51", PCI device ID: , UID: "1217d66b-e676-41db-aa9b-10e28fb7b299", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0521 19:45:48.252512 3793 cni.go:473] [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:af:74:45:cb:3a\",\"name\":\"d2165572207988b\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d1c550e2-6a8a-452c-a03a-a0f1d419fdbd\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:45:58.653943 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0521 19:46:04.359279 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0521 19:46:06.370623 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0521 19:46:13.127282 3793 obj_retry.go:498] Detected object openshift-console/console-84879d75bc-rdnjz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:13.127886 3793 obj_retry.go:498] Detected object openshift-console/console-84879d75bc-rdnjz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:13.481969 3793 obj_retry.go:492] Detected object openshift-console/console-84879d75bc-rdnjz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:13.481991 3793 obj_retry.go:492] Detected object openshift-console/console-84879d75bc-rdnjz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:13.489699 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84879d75bc-rdnjz I0521 19:46:13.489723 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84879d75bc-rdnjz I0521 19:46:17.342797 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0521 19:46:23.987556 3793 obj_retry.go:498] Detected object openshift-console/console-84879d75bc-5jjfm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:23.988108 3793 obj_retry.go:498] Detected object openshift-console/console-84879d75bc-5jjfm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:46:24.772044 3793 obj_retry.go:492] Detected object openshift-console/console-84879d75bc-5jjfm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:24.772077 3793 obj_retry.go:492] Detected object openshift-console/console-84879d75bc-5jjfm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:46:24.779789 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84879d75bc-5jjfm I0521 19:46:24.779818 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84879d75bc-5jjfm I0521 19:46:50.241770 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=142 I0521 19:46:53.370764 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0521 19:46:55.250477 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=707 I0521 19:48:15.332593 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0521 19:48:37.365457 3793 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0521 19:48:53.461616 3793 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-cztvj for pod on switch ip-10-0-2-49.ec2.internal I0521 19:48:53.461761 3793 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj I0521 19:48:53.489669 3793 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj] pod update took 27.948097ms I0521 19:48:53.489693 3793 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj] addLogicalPort annotation time took 27.97696ms I0521 19:48:53.490906 3793 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj] addLogicalPort took 29.304726ms, libovsdb time 855.305µs I0521 19:48:53.821524 3793 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj c10e180173614360e7002803dd7e12d580582efccf35e2ea6402544734e4cba7 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj c10e180173614360e7002803dd7e12d580582efccf35e2ea6402544734e4cba7 network default NAD default NAD key ] I0521 19:48:53.830794 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-cztvj, hostIfaceName: c10e18017361436, network: default, NAD default, SandboxID: "c10e180173614360e7002803dd7e12d580582efccf35e2ea6402544734e4cba7", PCI device ID: , UID: "2f0e7a8e-4940-45f3-a6a5-345da8265dae", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0521 19:48:53.882324 3793 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj c10e180173614360e7002803dd7e12d580582efccf35e2ea6402544734e4cba7 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-cztvj c10e180173614360e7002803dd7e12d580582efccf35e2ea6402544734e4cba7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:62:73:24:58:20\",\"name\":\"c10e18017361436\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2124689f-48d4-4fca-90e1-c73942fd7a8d\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:48:57.364300 3793 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:49:05.244691 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 W0521 19:49:08.370360 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0521 19:49:08.412910 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0521 19:49:08.412992 3793 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-7vfwj IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-7vfwj: no pod IPs found I0521 19:49:12.539334 3793 cni.go:452] [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key ] I0521 19:49:12.619141 3793 cni.go:473] [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-76df66f86b-blj9b d2165572207988bb8b0d50b2585f5e1b56b8caae62407a13f1a99ebf63c60b51 network default NAD default NAD key default], result "{}", err I0521 19:49:12.675069 3793 obj_retry.go:498] Detected object openshift-console/console-76df66f86b-blj9b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:12.675103 3793 pods.go:176] Deleting pod: openshift-console/console-76df66f86b-blj9b I0521 19:49:12.676354 3793 pods.go:236] Attempting to release IPs for pod: openshift-console/console-76df66f86b-blj9b, ips: 10.128.0.31 I0521 19:49:12.676402 3793 obj_retry.go:498] Detected object openshift-console/console-76df66f86b-blj9b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:12.683267 3793 base_network_controller_pods.go:487] [default/openshift-console/console-56d7ccd74c-rtkvp] creating logical port openshift-console_console-56d7ccd74c-rtkvp for pod on switch ip-10-0-2-49.ec2.internal I0521 19:49:12.683402 3793 kube.go:256] Updating pod openshift-console/console-56d7ccd74c-rtkvp I0521 19:49:12.699535 3793 pod.go:62] [openshift-console/console-56d7ccd74c-rtkvp] pod update took 16.179092ms I0521 19:49:12.699561 3793 base_network_controller_pods.go:951] [default/openshift-console/console-56d7ccd74c-rtkvp] addLogicalPort annotation time took 16.208237ms I0521 19:49:12.700723 3793 pods.go:271] [openshift-console/console-56d7ccd74c-rtkvp] addLogicalPort took 17.470714ms, libovsdb time 803.412µs I0521 19:49:13.041342 3793 cni.go:452] [openshift-console/console-56d7ccd74c-rtkvp 3f4060e71a2eedea40f0e8c72647cf140f27b129462497cc3e73fe48b5df79f8 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-56d7ccd74c-rtkvp 3f4060e71a2eedea40f0e8c72647cf140f27b129462497cc3e73fe48b5df79f8 network default NAD default NAD key ] I0521 19:49:13.051023 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-56d7ccd74c-rtkvp, hostIfaceName: 3f4060e71a2eede, network: default, NAD default, SandboxID: "3f4060e71a2eedea40f0e8c72647cf140f27b129462497cc3e73fe48b5df79f8", PCI device ID: , UID: "1d226c68-23bd-4fea-973b-15b85151a6e8", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0521 19:49:13.102660 3793 cni.go:473] [openshift-console/console-56d7ccd74c-rtkvp 3f4060e71a2eedea40f0e8c72647cf140f27b129462497cc3e73fe48b5df79f8 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-56d7ccd74c-rtkvp 3f4060e71a2eedea40f0e8c72647cf140f27b129462497cc3e73fe48b5df79f8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:f3:79:05:d2:59\",\"name\":\"3f4060e71a2eede\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2fbc9a87-7ebb-4047-ab14-0b69bdda81fa\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:49:13.448041 3793 obj_retry.go:492] Detected object openshift-console/console-76df66f86b-blj9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:13.448070 3793 obj_retry.go:492] Detected object openshift-console/console-76df66f86b-blj9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:13.456019 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-76df66f86b-blj9b I0521 19:49:13.456045 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-76df66f86b-blj9b W0521 19:49:23.486544 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0521 19:49:23.716826 3793 obj_retry.go:498] Detected object openshift-console/console-76df66f86b-gmrx2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:23.717409 3793 obj_retry.go:498] Detected object openshift-console/console-76df66f86b-gmrx2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:49:24.215498 3793 obj_retry.go:492] Detected object openshift-console/console-76df66f86b-gmrx2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:24.215520 3793 obj_retry.go:492] Detected object openshift-console/console-76df66f86b-gmrx2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:49:24.221879 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-76df66f86b-gmrx2 I0521 19:49:24.221926 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-76df66f86b-gmrx2 I0521 19:49:38.337544 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0521 19:49:53.353295 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0521 19:50:00.177430 3793 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29656550-h7fmd for pod on switch ip-10-0-2-49.ec2.internal I0521 19:50:00.177519 3793 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd I0521 19:50:00.194837 3793 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd] pod update took 17.339764ms I0521 19:50:00.194863 3793 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd] addLogicalPort annotation time took 17.372928ms I0521 19:50:00.196081 3793 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd] addLogicalPort took 18.668549ms, libovsdb time 866.752µs I0521 19:50:00.559708 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key ] I0521 19:50:00.569044 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29656550-h7fmd, hostIfaceName: 90ec636b3762c47, network: default, NAD default, SandboxID: "90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c", PCI device ID: , UID: "1897d59c-d691-497a-9d53-9e24190e7469", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0521 19:50:00.620891 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:34:e4:68:ac:e5\",\"name\":\"90ec636b3762c47\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae0df141-da7f-4ac7-a8d6-b748102a2b4e\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:50:03.650096 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key ] I0521 19:50:03.734839 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd 90ec636b3762c47d7158c4b2978bea82fd83cb674ab86df0b4e900a42eab073c network default NAD default NAD key default], result "{}", err I0521 19:50:03.779841 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:03.779896 3793 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd I0521 19:50:03.781076 3793 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd, ips: 10.128.0.34 I0521 19:50:03.781103 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:50:04.667754 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:04.667781 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656550-h7fmd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:50:12.332626 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0521 19:50:13.348802 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0521 19:50:30.242446 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0521 19:50:42.248535 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=17 I0521 19:50:52.244429 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=110 I0521 19:50:52.324946 3793 namespace.go:98] [tsf] adding namespace I0521 19:50:52.325911 3793 namespace.go:102] [tsf] adding namespace took 944.144µs I0521 19:50:52.348037 3793 namespace.go:142] [tsf] updating namespace I0521 19:50:52.361508 3793 namespace.go:142] [tsf] updating namespace I0521 19:50:56.080920 3793 namespace.go:98] [tsf-tas] adding namespace I0521 19:50:56.081786 3793 namespace.go:102] [tsf-tas] adding namespace took 845.904µs I0521 19:50:56.091698 3793 namespace.go:98] [tsf-tpa] adding namespace I0521 19:50:56.091736 3793 namespace.go:98] [rhtpa-operator] adding namespace I0521 19:50:56.091851 3793 namespace.go:98] [cert-manager-operator] adding namespace I0521 19:50:56.092012 3793 namespace.go:98] [rhbk-operator] adding namespace I0521 19:50:56.092078 3793 namespace.go:98] [tsf-keycloak] adding namespace I0521 19:50:56.092224 3793 namespace.go:98] [konflux-operator] adding namespace I0521 19:50:56.092630 3793 namespace.go:102] [tsf-tpa] adding namespace took 906.383µs I0521 19:50:56.093170 3793 namespace.go:102] [rhtpa-operator] adding namespace took 1.412345ms I0521 19:50:56.093197 3793 namespace.go:98] [konflux-ui] adding namespace I0521 19:50:56.093747 3793 namespace.go:102] [konflux-ui] adding namespace took 542.718µs I0521 19:50:56.094286 3793 namespace.go:102] [cert-manager-operator] adding namespace took 2.415292ms I0521 19:50:56.094785 3793 namespace.go:102] [rhbk-operator] adding namespace took 2.757987ms I0521 19:50:56.095282 3793 namespace.go:102] [tsf-keycloak] adding namespace took 3.188842ms I0521 19:50:56.095699 3793 namespace.go:102] [konflux-operator] adding namespace took 3.453462ms I0521 19:50:56.104676 3793 namespace.go:142] [tsf-tas] updating namespace I0521 19:50:56.123604 3793 namespace.go:142] [tsf-tas] updating namespace I0521 19:50:56.138509 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:50:56.138511 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:50:56.145632 3793 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:50:56.145751 3793 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:50:56.156423 3793 namespace.go:142] [rhbk-operator] updating namespace I0521 19:50:56.168491 3793 namespace.go:142] [konflux-operator] updating namespace I0521 19:50:56.168553 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:50:56.168671 3793 namespace.go:142] [tsf-tas] updating namespace I0521 19:50:56.197775 3793 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:50:56.210543 3793 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:50:56.219562 3793 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:50:56.241543 3793 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:50:56.280143 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:50:56.304475 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:50:56.334036 3793 namespace.go:142] [rhbk-operator] updating namespace I0521 19:50:56.356545 3793 namespace.go:142] [rhbk-operator] updating namespace I0521 19:50:56.360357 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:50:56.375714 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:50:56.391345 3793 namespace.go:142] [konflux-operator] updating namespace I0521 19:50:56.412872 3793 namespace.go:142] [konflux-operator] updating namespace I0521 19:50:56.433354 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:50:56.450409 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:50:57.266109 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:50:57.292911 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:50:58.798924 3793 base_network_controller_pods.go:487] [default/openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd] creating logical port openshift-marketplace_5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd for pod on switch ip-10-0-2-49.ec2.internal I0521 19:50:58.799030 3793 kube.go:256] Updating pod openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd I0521 19:50:58.812998 3793 pod.go:62] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd] pod update took 13.998299ms I0521 19:50:58.813018 3793 base_network_controller_pods.go:951] [default/openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd] addLogicalPort annotation time took 14.029459ms I0521 19:50:58.814272 3793 pods.go:271] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd] addLogicalPort took 15.371136ms, libovsdb time 824.015µs I0521 19:50:58.908296 3793 base_network_controller_pods.go:487] [default/openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv] creating logical port openshift-marketplace_79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv for pod on switch ip-10-0-2-49.ec2.internal I0521 19:50:58.908442 3793 kube.go:256] Updating pod openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv I0521 19:50:58.922399 3793 pod.go:62] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv] pod update took 13.985459ms I0521 19:50:58.922418 3793 base_network_controller_pods.go:951] [default/openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv] addLogicalPort annotation time took 14.009074ms I0521 19:50:58.923562 3793 pods.go:271] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv] addLogicalPort took 15.283247ms, libovsdb time 818.923µs I0521 19:50:59.156185 3793 cni.go:452] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key ] I0521 19:50:59.166275 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd, hostIfaceName: f7ef416c309cf6b, network: default, NAD default, SandboxID: "f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74", PCI device ID: , UID: "47e05709-2af2-4d51-aadf-a6e18e6a80ed", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0521 19:50:59.219698 3793 cni.go:473] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:84:a3:11:11:92\",\"name\":\"f7ef416c309cf6b\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6627f5b1-44fd-4862-96c5-46a121b1feda\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:50:59.266948 3793 cni.go:452] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key ] I0521 19:50:59.276679 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv, hostIfaceName: 755c7025e031c59, network: default, NAD default, SandboxID: "755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808", PCI device ID: , UID: "37e38417-cade-4b37-b3e9-ffb024d094c5", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0521 19:50:59.328141 3793 cni.go:473] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:36:89:ad:23:f2\",\"name\":\"755c7025e031c59\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7cc6009-da63-49bd-b807-827825ff93c9\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:01.872666 3793 base_network_controller_pods.go:487] [default/openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt] creating logical port openshift-marketplace_2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:01.872800 3793 kube.go:256] Updating pod openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt I0521 19:51:01.887914 3793 pod.go:62] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt] pod update took 15.143393ms I0521 19:51:01.887945 3793 base_network_controller_pods.go:951] [default/openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt] addLogicalPort annotation time took 15.180384ms I0521 19:51:01.889206 3793 pods.go:271] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt] addLogicalPort took 16.556839ms, libovsdb time 907.183µs I0521 19:51:02.233482 3793 cni.go:452] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key ] I0521 19:51:02.243179 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt, hostIfaceName: b984c447b323230, network: default, NAD default, SandboxID: "b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9", PCI device ID: , UID: "3ecf4afa-cafc-4735-a2d3-bc98b26e7b2d", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0521 19:51:02.293540 3793 cni.go:473] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:d4:3b:30:a3:d2\",\"name\":\"b984c447b323230\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/47db26d3-c817-408e-8ff7-a7f411bbfe21\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:02.919953 3793 cni.go:452] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key ] I0521 19:51:02.988806 3793 cni.go:473] [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd f7ef416c309cf6b186cc1d2017bcc4f4df3c6b302a2e3ea3500663537df89b74 network default NAD default NAD key default], result "{}", err I0521 19:51:03.027118 3793 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:03.027150 3793 pods.go:176] Deleting pod: openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd I0521 19:51:03.028677 3793 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd, ips: 10.128.0.35 I0521 19:51:03.028702 3793 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:03.028720 3793 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:03.028728 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:51:03.028912 3793 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:03.028927 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:51:03.592346 3793 base_network_controller_pods.go:487] [default/openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5] creating logical port openshift-marketplace_0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:03.592481 3793 kube.go:256] Updating pod openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 I0521 19:51:03.607105 3793 pod.go:62] [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5] pod update took 14.654173ms I0521 19:51:03.607127 3793 base_network_controller_pods.go:951] [default/openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5] addLogicalPort annotation time took 14.680449ms I0521 19:51:03.608466 3793 pods.go:271] [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5] addLogicalPort took 16.132138ms, libovsdb time 889.975µs I0521 19:51:03.929385 3793 cni.go:452] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key ] I0521 19:51:03.952706 3793 cni.go:452] [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key ] I0521 19:51:03.968608 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5, hostIfaceName: eb78080281ccd4c, network: default, NAD default, SandboxID: "eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e", PCI device ID: , UID: "a494f579-c10d-42d3-84a8-e6a0dc6cd1f0", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0521 19:51:03.999982 3793 cni.go:473] [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv 755c7025e031c596b7d8b3f64563752c9e4ccd6630f3485ef0e6b976c16eb808 network default NAD default NAD key default], result "{}", err I0521 19:51:04.019482 3793 cni.go:473] [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:d3:75:7e:14:f3\",\"name\":\"eb78080281ccd4c\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8c71a56e-0646-48d9-ac95-15a0c2231b72\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:04.049043 3793 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:04.049585 3793 pods.go:176] Deleting pod: openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv I0521 19:51:04.051047 3793 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv, ips: 10.128.0.36 I0521 19:51:04.051074 3793 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:04.051098 3793 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:04.051106 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:51:04.051314 3793 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:04.051331 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:51:04.055052 3793 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:04.055076 3793 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:04.055107 3793 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:04.055124 3793 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ekphqd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:05.053964 3793 base_network_controller_pods.go:487] [default/openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb] creating logical port openshift-marketplace_03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:05.054091 3793 kube.go:256] Updating pod openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb I0521 19:51:05.068706 3793 pod.go:62] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb] pod update took 14.639642ms I0521 19:51:05.068734 3793 base_network_controller_pods.go:951] [default/openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb] addLogicalPort annotation time took 14.671674ms I0521 19:51:05.070188 3793 pods.go:271] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb] addLogicalPort took 16.245034ms, libovsdb time 969.762µs I0521 19:51:05.077674 3793 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:05.077698 3793 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:05.077712 3793 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:05.077728 3793 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51nvbcv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:05.157875 3793 base_network_controller_pods.go:487] [default/openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx] creating logical port openshift-marketplace_09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:05.158003 3793 kube.go:256] Updating pod openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx I0521 19:51:05.172955 3793 pod.go:62] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx] pod update took 14.980828ms I0521 19:51:05.172983 3793 base_network_controller_pods.go:951] [default/openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx] addLogicalPort annotation time took 15.011756ms I0521 19:51:05.174343 3793 pods.go:271] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx] addLogicalPort took 16.490126ms, libovsdb time 819.059µs I0521 19:51:05.432752 3793 cni.go:452] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key ] I0521 19:51:05.443040 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb, hostIfaceName: a811847a87b4d42, network: default, NAD default, SandboxID: "a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a", PCI device ID: , UID: "6c73958a-0ad4-43a4-9c1e-6a060bd17569", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0521 19:51:05.497816 3793 cni.go:473] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:34:c0:ba:fe:d6\",\"name\":\"a811847a87b4d42\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d2e4ac0e-6449-41c8-a0a4-b0292067e814\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:05.521201 3793 cni.go:452] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key ] ADD starting CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key ] I0521 19:51:05.549022 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-marketplace, podName: 09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx, hostIfaceName: 22b78d047479e2a, network: default, NAD default, SandboxID: "22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0", PCI device ID: , UID: "8b907381-9f3b-4acb-9f9a-c4e79725bec4", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0521 19:51:05.604562 3793 cni.go:473] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key default] ADD finished CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:1a:bd:00:63:44\",\"name\":\"22b78d047479e2a\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fba2851f-cffc-4bf5-9fbe-f5bf58762fa4\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:07.961885 3793 cni.go:452] [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key ] I0521 19:51:07.962595 3793 cni.go:452] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key ] I0521 19:51:08.034693 3793 cni.go:473] [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 eb78080281ccd4c08a9a35a913818c3dd996191d4935dcc1c3a423a18b453b7e network default NAD default NAD key default], result "{}", err I0521 19:51:08.067291 3793 cni.go:473] [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt b984c447b323230ceb11fe118621e998744d84ec55c9f371d869e349193ea7a9 network default NAD default NAD key default], result "{}", err I0521 19:51:08.090719 3793 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.090752 3793 pods.go:176] Deleting pod: openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 I0521 19:51:08.092385 3793 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5, ips: 10.128.0.38 I0521 19:51:08.092420 3793 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.092438 3793 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.092445 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:51:08.092647 3793 obj_retry.go:498] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.092659 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:51:08.126789 3793 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.126825 3793 pods.go:176] Deleting pod: openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt I0521 19:51:08.128546 3793 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt, ips: 10.128.0.37 I0521 19:51:08.128573 3793 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.128595 3793 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.128604 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:51:08.128791 3793 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:08.128812 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:51:09.124537 3793 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.124562 3793 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.124576 3793 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.124587 3793 obj_retry.go:492] Detected object openshift-marketplace/0d4594801d1bcc50dfd40d9aaf87fe5abcddd80fc031fef403b56c9a1dk6wc5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.160287 3793 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.160326 3793 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.160338 3793 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.160348 3793 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dnkrxt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:09.979445 3793 cni.go:452] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key ] I0521 19:51:09.980164 3793 cni.go:452] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key ] DEL starting CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key ] I0521 19:51:10.063196 3793 cni.go:473] [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx 22b78d047479e2ae254e2450ef0670be6a801583dbd3f0262f83f2e5c240aaa0 network default NAD default NAD key default], result "{}", err I0521 19:51:10.078899 3793 cni.go:473] [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key default] DEL finished CNI request [openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb a811847a87b4d4274a83a497feb562e5400e8e15d7a5ed2885596fdebabe163a network default NAD default NAD key default], result "{}", err I0521 19:51:10.135248 3793 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.135283 3793 pods.go:176] Deleting pod: openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx I0521 19:51:10.136644 3793 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx, ips: 10.128.0.40 I0521 19:51:10.136667 3793 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.136685 3793 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.136692 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:51:10.136854 3793 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.136869 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:51:10.166500 3793 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.166537 3793 pods.go:176] Deleting pod: openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb I0521 19:51:10.167938 3793 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb, ips: 10.128.0.39 I0521 19:51:10.167968 3793 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.167986 3793 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.167993 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0521 19:51:10.168209 3793 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0521 19:51:10.168227 3793 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0521 19:51:11.166454 3793 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.166477 3793 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.166491 3793 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.166505 3793 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7p9znx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.198851 3793 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.198874 3793 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.198887 3793 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:11.198898 3793 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136dxwxb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:12.667335 3793 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-mztgw for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:12.667462 3793 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw I0521 19:51:12.686735 3793 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw] pod update took 19.296779ms I0521 19:51:12.686756 3793 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw] addLogicalPort annotation time took 19.324151ms I0521 19:51:12.688042 3793 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw] addLogicalPort took 20.726818ms, libovsdb time 875.039µs I0521 19:51:13.028275 3793 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw eab38178ccf8f28f5267b1aefbdfbdf51e2125a363b13af9fafbac22a219ee38 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw eab38178ccf8f28f5267b1aefbdfbdf51e2125a363b13af9fafbac22a219ee38 network default NAD default NAD key ] I0521 19:51:13.037574 3793 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-mztgw, hostIfaceName: eab38178ccf8f28, network: default, NAD default, SandboxID: "eab38178ccf8f28f5267b1aefbdfbdf51e2125a363b13af9fafbac22a219ee38", PCI device ID: , UID: "cf95e5d7-1804-4bd1-92a6-ccccfb516c09", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0521 19:51:13.088565 3793 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw eab38178ccf8f28f5267b1aefbdfbdf51e2125a363b13af9fafbac22a219ee38 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-mztgw eab38178ccf8f28f5267b1aefbdfbdf51e2125a363b13af9fafbac22a219ee38 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:ea:d5:b2:1a:bf\",\"name\":\"eab38178ccf8f28\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/223c92db-253e-4e82-af7e-52b7ff72c1f6\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:18.718597 3793 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-8qbr6 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:18.718757 3793 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6 I0521 19:51:18.746052 3793 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6] pod update took 27.324343ms I0521 19:51:18.746074 3793 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6] addLogicalPort annotation time took 27.351988ms I0521 19:51:18.747272 3793 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6] addLogicalPort took 28.697706ms, libovsdb time 830.248µs I0521 19:51:19.079745 3793 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6 996cf53693ec0ab53a0976c378894f77076d983f58087406ade8a234345dd284 network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6 996cf53693ec0ab53a0976c378894f77076d983f58087406ade8a234345dd284 network default NAD default NAD key ] I0521 19:51:19.089029 3793 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-8qbr6, hostIfaceName: 996cf53693ec0ab, network: default, NAD default, SandboxID: "996cf53693ec0ab53a0976c378894f77076d983f58087406ade8a234345dd284", PCI device ID: , UID: "6380b727-ee86-4c6a-9959-b38e5ab1c128", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0521 19:51:19.140938 3793 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6 996cf53693ec0ab53a0976c378894f77076d983f58087406ade8a234345dd284 network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-8qbr6 996cf53693ec0ab53a0976c378894f77076d983f58087406ade8a234345dd284 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:9c:e1:bc:0d:96\",\"name\":\"996cf53693ec0ab\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/355aad5c-04c6-48e3-8be1-6c45db507be8\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:51:22.698398 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0521 19:51:22.732981 3793 namespace.go:98] [cert-manager] adding namespace I0521 19:51:22.733880 3793 namespace.go:102] [cert-manager] adding namespace took 878.9µs I0521 19:51:22.920704 3793 namespace.go:142] [cert-manager] updating namespace I0521 19:51:22.975631 3793 namespace.go:142] [cert-manager] updating namespace W0521 19:51:23.162628 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0521 19:51:24.112407 3793 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-jv6h4 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:24.112532 3793 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4 I0521 19:51:24.198235 3793 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4] pod update took 85.728396ms I0521 19:51:24.198258 3793 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4] addLogicalPort annotation time took 85.758266ms I0521 19:51:24.199479 3793 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4] addLogicalPort took 87.090249ms, libovsdb time 854.552µs I0521 19:51:25.224797 3793 namespace.go:142] [openshift-operators] updating namespace I0521 19:51:25.259755 3793 namespace.go:142] [openshift-operators] updating namespace I0521 19:51:25.819498 3793 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr] creating logical port rhbk-operator_rhbk-operator-6fc7dbc666-n7ppr for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:25.819630 3793 kube.go:256] Updating pod rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr I0521 19:51:25.844922 3793 pod.go:62] [rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr] pod update took 25.317659ms I0521 19:51:25.844944 3793 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr] addLogicalPort annotation time took 25.342863ms I0521 19:51:25.846269 3793 pods.go:271] [rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr] addLogicalPort took 26.789671ms, libovsdb time 931.123µs I0521 19:51:26.179829 3793 cni.go:452] [rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr 64e5ba26dbfda479d3568809ed40f54a27c15a93db7e74f7d7044fced9f71fa7 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr 64e5ba26dbfda479d3568809ed40f54a27c15a93db7e74f7d7044fced9f71fa7 network default NAD default NAD key ] I0521 19:51:26.189196 3793 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-6fc7dbc666-n7ppr, hostIfaceName: 64e5ba26dbfda47, network: default, NAD default, SandboxID: "64e5ba26dbfda479d3568809ed40f54a27c15a93db7e74f7d7044fced9f71fa7", PCI device ID: , UID: "fe109e78-04fd-437c-84a5-7b67f80e7310", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0521 19:51:26.241544 3793 cni.go:473] [rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr 64e5ba26dbfda479d3568809ed40f54a27c15a93db7e74f7d7044fced9f71fa7 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-6fc7dbc666-n7ppr 64e5ba26dbfda479d3568809ed40f54a27c15a93db7e74f7d7044fced9f71fa7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:ab:76:a1:f4:73\",\"name\":\"64e5ba26dbfda47\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ef91f8a-f60a-4319-a6a9-31e8dce3056a\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:51:26.257032 3793 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4 09d14da8916e77f0d6eddd0c4b84524f2714b5291cba81404c27bd910ae63b2d network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4 09d14da8916e77f0d6eddd0c4b84524f2714b5291cba81404c27bd910ae63b2d network default NAD default NAD key ] I0521 19:51:26.266950 3793 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-jv6h4, hostIfaceName: 09d14da8916e77f, network: default, NAD default, SandboxID: "09d14da8916e77f0d6eddd0c4b84524f2714b5291cba81404c27bd910ae63b2d", PCI device ID: , UID: "5d7975c0-02b9-4658-9471-324e07b0cb04", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0521 19:51:26.317914 3793 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4 09d14da8916e77f0d6eddd0c4b84524f2714b5291cba81404c27bd910ae63b2d network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-jv6h4 09d14da8916e77f0d6eddd0c4b84524f2714b5291cba81404c27bd910ae63b2d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:f7:5a:77:a4:ee\",\"name\":\"09d14da8916e77f\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5ee791e-50f4-4e6b-a4fb-9b527ab50699\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:51:30.155619 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0521 19:51:30.155642 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0521 19:51:33.745513 3793 namespace.go:142] [openshift-operators] updating namespace W0521 19:51:34.073036 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0521 19:51:34.266701 3793 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-j5ssw for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:34.266849 3793 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw I0521 19:51:34.295179 3793 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw] pod update took 28.357183ms I0521 19:51:34.295203 3793 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw] addLogicalPort annotation time took 28.384721ms I0521 19:51:34.296563 3793 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw] addLogicalPort took 29.878699ms, libovsdb time 903.936µs I0521 19:51:34.627097 3793 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw c09bd17b01a6e0295a90ea27a088934aa8079a7039b346ac50154a8e3d0e3ecd network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw c09bd17b01a6e0295a90ea27a088934aa8079a7039b346ac50154a8e3d0e3ecd network default NAD default NAD key ] I0521 19:51:34.636739 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-j5ssw, hostIfaceName: c09bd17b01a6e02, network: default, NAD default, SandboxID: "c09bd17b01a6e0295a90ea27a088934aa8079a7039b346ac50154a8e3d0e3ecd", PCI device ID: , UID: "eb755b2d-68ad-41e9-a6e6-e2b45a2cac5a", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0521 19:51:34.689425 3793 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw c09bd17b01a6e0295a90ea27a088934aa8079a7039b346ac50154a8e3d0e3ecd network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-j5ssw c09bd17b01a6e0295a90ea27a088934aa8079a7039b346ac50154a8e3d0e3ecd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:95:70:17:14:f0\",\"name\":\"c09bd17b01a6e02\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a224912f-becd-4dd3-bd57-34780cd19fe6\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:51:36.130874 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook W0521 19:51:36.130899 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook I0521 19:51:42.820140 3793 namespace.go:98] [trusted-artifact-signer] adding namespace I0521 19:51:42.820943 3793 namespace.go:102] [trusted-artifact-signer] adding namespace took 778.744µs I0521 19:51:42.866936 3793 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:51:42.881210 3793 namespace.go:142] [trusted-artifact-signer] updating namespace W0521 19:51:43.068524 3793 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0521 19:51:49.331417 3793 namespace.go:98] [openshift-pipelines] adding namespace I0521 19:51:49.332245 3793 namespace.go:102] [openshift-pipelines] adding namespace took 799.437µs I0521 19:51:49.354985 3793 namespace.go:142] [openshift-pipelines] updating namespace I0521 19:51:50.061619 3793 namespace.go:142] [cert-manager] updating namespace I0521 19:51:50.262876 3793 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:51:50.662945 3793 namespace.go:142] [konflux-operator] updating namespace I0521 19:51:50.867191 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:51:50.968422 3793 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:50.969030 3793 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:51:51.002873 3793 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:51.002894 3793 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:51:51.012946 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0521 19:51:51.012968 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0521 19:51:51.062802 3793 namespace.go:142] [open-cluster-management-2qegovcgb34vgjgrmsm5lc08sjlrclho] updating namespace I0521 19:51:51.467121 3793 namespace.go:142] [rhbk-operator] updating namespace I0521 19:51:51.667337 3793 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:51:51.867153 3793 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:51:52.071092 3793 namespace.go:142] [tsf] updating namespace I0521 19:51:52.282506 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:51:52.474036 3793 namespace.go:142] [tsf-tas] updating namespace I0521 19:51:52.672938 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:51:52.803090 3793 namespace.go:142] [cert-manager] updating namespace I0521 19:51:52.813188 3793 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:51:52.823270 3793 namespace.go:142] [default] updating namespace I0521 19:51:52.834167 3793 namespace.go:142] [konflux-operator] updating namespace I0521 19:51:52.842579 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:51:52.856470 3793 namespace.go:142] [open-cluster-management-2qegovcgb34vgjgrmsm5lc08sjlrclho] updating namespace I0521 19:51:52.865893 3793 namespace.go:142] [openshift] updating namespace I0521 19:51:52.876796 3793 namespace.go:142] [rhbk-operator] updating namespace I0521 19:51:52.888188 3793 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:51:52.896333 3793 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:51:52.906345 3793 namespace.go:142] [tsf] updating namespace I0521 19:51:52.916445 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:51:52.933955 3793 namespace.go:142] [tsf-tas] updating namespace I0521 19:51:52.954594 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:51:53.055417 3793 namespace.go:142] [cert-manager] updating namespace I0521 19:51:53.156520 3793 namespace.go:142] [cert-manager-operator] updating namespace I0521 19:51:53.255880 3793 namespace.go:142] [default] updating namespace I0521 19:51:53.359060 3793 namespace.go:142] [konflux-operator] updating namespace I0521 19:51:53.458721 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:51:53.556512 3793 namespace.go:142] [open-cluster-management-2qegovcgb34vgjgrmsm5lc08sjlrclho] updating namespace I0521 19:51:53.660921 3793 namespace.go:142] [openshift] updating namespace I0521 19:51:53.757212 3793 namespace.go:142] [rhbk-operator] updating namespace I0521 19:51:53.855316 3793 namespace.go:142] [rhtpa-operator] updating namespace I0521 19:51:53.957845 3793 namespace.go:142] [trusted-artifact-signer] updating namespace I0521 19:51:54.058386 3793 namespace.go:142] [tsf] updating namespace I0521 19:51:54.158453 3793 namespace.go:142] [tsf-keycloak] updating namespace I0521 19:51:54.254990 3793 namespace.go:142] [tsf-tas] updating namespace I0521 19:51:54.354464 3793 namespace.go:142] [tsf-tpa] updating namespace I0521 19:51:55.014410 3793 namespace.go:142] [openshift-pipelines] updating namespace I0521 19:51:55.345594 3793 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-f9plq for pod on switch ip-10-0-2-49.ec2.internal I0521 19:51:55.345698 3793 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq I0521 19:51:55.403345 3793 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq] pod update took 57.669006ms I0521 19:51:55.403381 3793 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq] addLogicalPort annotation time took 57.695488ms I0521 19:51:55.404483 3793 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq] addLogicalPort took 58.909125ms, libovsdb time 733.135µs W0521 19:52:01.873504 3793 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0521 19:52:03.579166 3793 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq c12487d0baa09a33f7e47e5a3a10bd9e4b0167675acefd8babd5cd13722b5956 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq c12487d0baa09a33f7e47e5a3a10bd9e4b0167675acefd8babd5cd13722b5956 network default NAD default NAD key ] I0521 19:52:03.588639 3793 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-f9plq, hostIfaceName: c12487d0baa09a3, network: default, NAD default, SandboxID: "c12487d0baa09a33f7e47e5a3a10bd9e4b0167675acefd8babd5cd13722b5956", PCI device ID: , UID: "8fe9e003-aa05-4425-b0b1-5f15d8c9d982", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0521 19:52:03.641506 3793 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq c12487d0baa09a33f7e47e5a3a10bd9e4b0167675acefd8babd5cd13722b5956 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-f9plq c12487d0baa09a33f7e47e5a3a10bd9e4b0167675acefd8babd5cd13722b5956 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:25:46:d9:d3:1c\",\"name\":\"c12487d0baa09a3\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7be7f5dc-83b7-4c72-8959-585afc79d700\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:52:04.283815 3793 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-wkbz4 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:52:04.283937 3793 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4 I0521 19:52:04.299109 3793 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4] pod update took 15.198913ms I0521 19:52:04.299130 3793 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4] addLogicalPort annotation time took 15.224501ms I0521 19:52:04.300480 3793 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4] addLogicalPort took 16.680782ms, libovsdb time 870.477µs I0521 19:52:04.694224 3793 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4 2642c0e32a08d5c0b32e9143059e66769804602244bba7b516ccdd7f8b509a61 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4 2642c0e32a08d5c0b32e9143059e66769804602244bba7b516ccdd7f8b509a61 network default NAD default NAD key ] I0521 19:52:04.706436 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-wkbz4, hostIfaceName: 2642c0e32a08d5c, network: default, NAD default, SandboxID: "2642c0e32a08d5c0b32e9143059e66769804602244bba7b516ccdd7f8b509a61", PCI device ID: , UID: "222ae11c-d477-47f5-b1eb-9b7499cd08b7", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0521 19:52:04.772074 3793 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4 2642c0e32a08d5c0b32e9143059e66769804602244bba7b516ccdd7f8b509a61 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-wkbz4 2642c0e32a08d5c0b32e9143059e66769804602244bba7b516ccdd7f8b509a61 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:31:8f:0a:18:f7\",\"name\":\"2642c0e32a08d5c\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d487374f-44d8-46f0-84d4-31356b3f187a\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:52:07.171184 3793 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0521 19:52:07.171212 3793 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0521 19:52:07.171218 3793 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0521 19:52:10.308620 3793 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0521 19:52:10.337050 3793 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:10.337070 3793 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:10.337076 3793 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:10.337080 3793 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:17.219423 3793 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:52:17.219442 3793 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:52:17.219447 3793 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0521 19:52:21.322733 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:21.322754 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:21.322760 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0521 19:52:21.322764 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0521 19:52:30.253822 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=250 W0521 19:52:31.316709 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0521 19:52:33.157423 3793 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:52:33.157922 3793 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:52:33.178953 3793 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:52:33.178970 3793 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:52:33.182971 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0521 19:52:33.182995 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0521 19:52:34.560639 3793 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-b2jvg for pod on switch ip-10-0-2-49.ec2.internal I0521 19:52:34.560770 3793 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg I0521 19:52:34.575794 3793 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg] pod update took 15.052551ms I0521 19:52:34.575815 3793 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg] addLogicalPort annotation time took 15.080913ms I0521 19:52:34.577040 3793 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg] addLogicalPort took 16.418764ms, libovsdb time 833.931µs I0521 19:52:34.920875 3793 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg 2ff446266c5eeb09ec9d9a7909268e86eaa33dd70a704ef9ba0f41b5285dd92e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg 2ff446266c5eeb09ec9d9a7909268e86eaa33dd70a704ef9ba0f41b5285dd92e network default NAD default NAD key ] I0521 19:52:34.930292 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-b2jvg, hostIfaceName: 2ff446266c5eeb0, network: default, NAD default, SandboxID: "2ff446266c5eeb09ec9d9a7909268e86eaa33dd70a704ef9ba0f41b5285dd92e", PCI device ID: , UID: "cdca349c-7c04-4593-b742-5a6ea9742a4a", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0521 19:52:34.983981 3793 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg 2ff446266c5eeb09ec9d9a7909268e86eaa33dd70a704ef9ba0f41b5285dd92e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-b2jvg 2ff446266c5eeb09ec9d9a7909268e86eaa33dd70a704ef9ba0f41b5285dd92e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:cb:fc:67:ab:a3\",\"name\":\"2ff446266c5eeb0\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2b305ec6-ef66-4c85-bf03-3e7f71003235\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:52:35.098176 3793 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0521 19:52:35.098876 3793 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0521 19:52:35.098895 3793 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0521 19:52:35.101272 3793 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0521 19:52:35.453120 3793 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:52:35.453216 3793 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0521 19:52:35.472546 3793 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 19.35997ms I0521 19:52:35.472568 3793 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 19.385015ms I0521 19:52:35.473838 3793 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 20.733848ms, libovsdb time 899.115µs I0521 19:52:35.816695 3793 cni.go:452] [tsf-keycloak/keycloak-0 d1fd2b6538a37ccd9030b0072a692b2f7f1e62af9bc58ed74a581bcf8e425741 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 d1fd2b6538a37ccd9030b0072a692b2f7f1e62af9bc58ed74a581bcf8e425741 network default NAD default NAD key ] I0521 19:52:35.826646 3793 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: d1fd2b6538a37cc, network: default, NAD default, SandboxID: "d1fd2b6538a37ccd9030b0072a692b2f7f1e62af9bc58ed74a581bcf8e425741", PCI device ID: , UID: "1530ed34-d2e1-4b41-b60d-a985d33262dd", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0521 19:52:35.879867 3793 cni.go:473] [tsf-keycloak/keycloak-0 d1fd2b6538a37ccd9030b0072a692b2f7f1e62af9bc58ed74a581bcf8e425741 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 d1fd2b6538a37ccd9030b0072a692b2f7f1e62af9bc58ed74a581bcf8e425741 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:30:16:c7:f4:2c\",\"name\":\"d1fd2b6538a37cc\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6856c901-21c7-4722-810e-a20bbcd7a0d2\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:52:39.066635 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0521 19:52:44.495823 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0521 19:52:44.495850 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0521 19:52:44.501222 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0521 19:52:44.501241 3793 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0521 19:52:56.138806 3793 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0521 19:53:11.057001 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0521 19:53:11.057029 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0521 19:53:11.061037 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0521 19:53:11.061058 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service I0521 19:53:15.358922 3793 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-jc4vc] creating logical port tsf-keycloak_tsf-iam-jc4vc for pod on switch ip-10-0-2-49.ec2.internal I0521 19:53:15.359040 3793 kube.go:256] Updating pod tsf-keycloak/tsf-iam-jc4vc I0521 19:53:15.375040 3793 pod.go:62] [tsf-keycloak/tsf-iam-jc4vc] pod update took 16.039707ms I0521 19:53:15.375063 3793 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-jc4vc] addLogicalPort annotation time took 16.068934ms I0521 19:53:15.376480 3793 pods.go:271] [tsf-keycloak/tsf-iam-jc4vc] addLogicalPort took 17.579045ms, libovsdb time 984.606µs I0521 19:53:15.721585 3793 cni.go:452] [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key ] I0521 19:53:15.730917 3793 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-jc4vc, hostIfaceName: ac52d364d480355, network: default, NAD default, SandboxID: "ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa", PCI device ID: , UID: "4870fc8f-c782-4cbb-871a-0083f10ecea1", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0521 19:53:15.785236 3793 cni.go:473] [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:82:10:b9:b2:a5\",\"name\":\"ac52d364d480355\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7951e249-0abf-4846-94e3-f63eb818d831\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:53:18.399186 3793 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:18.399845 3793 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:18.439423 3793 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:18.439454 3793 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:18.447231 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0521 19:53:18.447260 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0521 19:53:23.541573 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0521 19:53:23.541602 3793 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0521 19:53:24.307098 3793 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0521 19:53:24.307122 3793 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0521 19:53:24.307127 3793 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0521 19:53:29.576007 3793 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0521 19:53:34.811696 3793 cni.go:452] [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key ] I0521 19:53:34.892578 3793 cni.go:473] [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-jc4vc ac52d364d480355b38c5f183217c5a59e43b7ab3838f3d6f7c5ad81e4888d8aa network default NAD default NAD key default], result "{}", err I0521 19:53:34.950812 3793 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-jc4vc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:34.950842 3793 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-jc4vc I0521 19:53:34.952461 3793 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-jc4vc, ips: 10.128.0.50 I0521 19:53:34.952484 3793 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-jc4vc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0521 19:53:35.397891 3793 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0521 19:53:35.836625 3793 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-jc4vc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:35.836656 3793 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-jc4vc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:36.524148 3793 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-gn8fs-4c6bm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:36.524788 3793 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-gn8fs-4c6bm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:53:37.245768 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=92 I0521 19:53:37.393919 3793 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-gn8fs-4c6bm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:53:37.393940 3793 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-gn8fs-4c6bm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:53:38.392039 3793 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0521 19:53:41.382566 3793 namespace.go:142] [openshift-pipelines] updating namespace I0521 19:53:52.374567 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0521 19:53:56.345991 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0521 19:53:57.123599 3793 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-wcmxd for pod on switch ip-10-0-2-49.ec2.internal I0521 19:53:57.123683 3793 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd I0521 19:53:57.140504 3793 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd] pod update took 16.839137ms I0521 19:53:57.140529 3793 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd] addLogicalPort annotation time took 16.872244ms I0521 19:53:57.141983 3793 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd] addLogicalPort took 18.400539ms, libovsdb time 999.801µs I0521 19:53:57.219871 3793 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-hmttk for pod on switch ip-10-0-2-49.ec2.internal I0521 19:53:57.219998 3793 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk I0521 19:53:57.235504 3793 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk] pod update took 15.527131ms I0521 19:53:57.235527 3793 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk] addLogicalPort annotation time took 15.553651ms I0521 19:53:57.236781 3793 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk] addLogicalPort took 16.927686ms, libovsdb time 848.991µs I0521 19:53:57.485509 3793 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd 1c7f9aa1a284e512f284a06ece7583715ccc694991d2099bfed0c5dda5110c30 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd 1c7f9aa1a284e512f284a06ece7583715ccc694991d2099bfed0c5dda5110c30 network default NAD default NAD key ] I0521 19:53:57.494856 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-wcmxd, hostIfaceName: 1c7f9aa1a284e51, network: default, NAD default, SandboxID: "1c7f9aa1a284e512f284a06ece7583715ccc694991d2099bfed0c5dda5110c30", PCI device ID: , UID: "96a648f4-e020-4564-9104-91deb0ead33c", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0521 19:53:57.550239 3793 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd 1c7f9aa1a284e512f284a06ece7583715ccc694991d2099bfed0c5dda5110c30 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-wcmxd 1c7f9aa1a284e512f284a06ece7583715ccc694991d2099bfed0c5dda5110c30 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:0c:8c:26:92:8a\",\"name\":\"1c7f9aa1a284e51\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3bbed2c5-15e9-42d6-b23c-62317cc90023\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:53:57.581153 3793 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk 10c6d488b88ea14a672ca19b358789e5d2274652d221408e6a60588f9c356463 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk 10c6d488b88ea14a672ca19b358789e5d2274652d221408e6a60588f9c356463 network default NAD default NAD key ] I0521 19:53:57.590844 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-hmttk, hostIfaceName: 10c6d488b88ea14, network: default, NAD default, SandboxID: "10c6d488b88ea14a672ca19b358789e5d2274652d221408e6a60588f9c356463", PCI device ID: , UID: "e620dec6-e65c-4493-a0bc-b871193e3369", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0521 19:53:57.643993 3793 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk 10c6d488b88ea14a672ca19b358789e5d2274652d221408e6a60588f9c356463 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-hmttk 10c6d488b88ea14a672ca19b358789e5d2274652d221408e6a60588f9c356463 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:4a:1d:7a:3f:b3\",\"name\":\"10c6d488b88ea14\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e317bb2-41e7-43c2-b492-bc7042bf6113\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:54:00.961584 3793 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0521 19:54:00.961599 3793 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0521 19:54:00.985627 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0521 19:54:00.985643 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0521 19:54:01.015466 3793 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0521 19:54:10.724074 3793 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:10.724660 3793 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:10.745087 3793 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:10.745108 3793 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:10.751223 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0521 19:54:10.751248 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas W0521 19:54:11.969319 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0521 19:54:18.374550 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0521 19:54:19.752965 3793 obj_retry.go:498] Detected object tsf/patch-tekton-config-h86bk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:19.753557 3793 obj_retry.go:498] Detected object tsf/patch-tekton-config-h86bk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:20.775466 3793 obj_retry.go:492] Detected object tsf/patch-tekton-config-h86bk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:20.775488 3793 obj_retry.go:492] Detected object tsf/patch-tekton-config-h86bk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:28.010501 3793 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-mzcs5-wgj77 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:28.011026 3793 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-mzcs5-wgj77 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:28.891943 3793 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-mzcs5-wgj77 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:28.891969 3793 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-mzcs5-wgj77 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:32.959112 3793 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-85974789fd-cqb2s] creating logical port tsf-tas_rekor-search-ui-85974789fd-cqb2s for pod on switch ip-10-0-2-49.ec2.internal I0521 19:54:32.959204 3793 kube.go:256] Updating pod tsf-tas/rekor-search-ui-85974789fd-cqb2s I0521 19:54:32.972059 3793 pod.go:62] [tsf-tas/rekor-search-ui-85974789fd-cqb2s] pod update took 12.878122ms I0521 19:54:32.972091 3793 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-85974789fd-cqb2s] addLogicalPort annotation time took 12.913727ms I0521 19:54:32.973554 3793 pods.go:271] [tsf-tas/rekor-search-ui-85974789fd-cqb2s] addLogicalPort took 14.459012ms, libovsdb time 1.014319ms I0521 19:54:33.320013 3793 cni.go:452] [tsf-tas/rekor-search-ui-85974789fd-cqb2s fed842698619c12d6fb96a8c9beb4f08f7faf9a8cc14f6ced11dbaec9fa223f8 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-85974789fd-cqb2s fed842698619c12d6fb96a8c9beb4f08f7faf9a8cc14f6ced11dbaec9fa223f8 network default NAD default NAD key ] I0521 19:54:33.329173 3793 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-85974789fd-cqb2s, hostIfaceName: fed842698619c12, network: default, NAD default, SandboxID: "fed842698619c12d6fb96a8c9beb4f08f7faf9a8cc14f6ced11dbaec9fa223f8", PCI device ID: , UID: "6acb1139-7305-41b9-a326-432d42970cfb", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0521 19:54:33.382598 3793 cni.go:473] [tsf-tas/rekor-search-ui-85974789fd-cqb2s fed842698619c12d6fb96a8c9beb4f08f7faf9a8cc14f6ced11dbaec9fa223f8 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-85974789fd-cqb2s fed842698619c12d6fb96a8c9beb4f08f7faf9a8cc14f6ced11dbaec9fa223f8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:f5:f8:9d:0b:8a\",\"name\":\"fed842698619c12\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/72f3571e-4101-44bf-878b-82310dbd69ba\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:54:33.849665 3793 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-fxp46 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:33.850208 3793 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-fxp46 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:34.871649 3793 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-fxp46 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:34.871673 3793 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-fxp46 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:34.909743 3793 obj_retry.go:492] Detected object tsf/patch-tekton-config-h86bk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:34.909770 3793 obj_retry.go:492] Detected object tsf/patch-tekton-config-h86bk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:34.914991 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-h86bk I0521 19:54:34.915022 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-h86bk W0521 19:54:37.165190 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0521 19:54:38.874633 3793 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:38.875181 3793 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:38.895575 3793 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:38.895596 3793 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:38.902443 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0521 19:54:38.902467 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0521 19:54:38.936662 3793 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0521 19:54:42.089087 3793 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-859xc-rrbbm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:42.089665 3793 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-859xc-rrbbm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:42.969412 3793 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-859xc-rrbbm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:42.969435 3793 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-859xc-rrbbm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:44.917923 3793 obj_retry.go:498] Detected object tsf-tpa/create-db-nksbr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:44.918487 3793 obj_retry.go:498] Detected object tsf-tpa/create-db-nksbr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:45.941423 3793 obj_retry.go:492] Detected object tsf-tpa/create-db-nksbr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:45.941449 3793 obj_retry.go:492] Detected object tsf-tpa/create-db-nksbr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:54:47.990073 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0521 19:54:49.065815 3793 obj_retry.go:498] Detected object tsf-tpa/migrate-db-zr7dw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:49.066418 3793 obj_retry.go:498] Detected object tsf-tpa/migrate-db-zr7dw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0521 19:54:49.176174 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui W0521 19:54:49.999322 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0521 19:54:50.088190 3793 obj_retry.go:492] Detected object tsf-tpa/migrate-db-zr7dw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:50.088211 3793 obj_retry.go:492] Detected object tsf-tpa/migrate-db-zr7dw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:55.151797 3793 obj_retry.go:498] Detected object tsf-tpa/create-importers-9mdp2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:55.152318 3793 obj_retry.go:498] Detected object tsf-tpa/create-importers-9mdp2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:54:56.174202 3793 obj_retry.go:492] Detected object tsf-tpa/create-importers-9mdp2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:54:56.174235 3793 obj_retry.go:492] Detected object tsf-tpa/create-importers-9mdp2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0521 19:54:57.876625 3793 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0521 19:55:06.336522 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0521 19:55:13.259062 3793 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-rh69p-znjqh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:55:13.259582 3793 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-rh69p-znjqh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:55:14.283843 3793 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-rh69p-znjqh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:55:14.283870 3793 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-rh69p-znjqh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:55:26.154614 3793 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:55:26.155205 3793 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:55:26.202143 3793 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:55:26.202164 3793 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:55:26.209013 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0521 19:55:26.209028 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0521 19:55:26.972382 3793 namespace.go:98] [image-controller] adding namespace I0521 19:55:26.973299 3793 namespace.go:102] [image-controller] adding namespace took 898.478µs I0521 19:55:26.985671 3793 namespace.go:98] [build-service] adding namespace I0521 19:55:26.985746 3793 namespace.go:98] [integration-service] adding namespace I0521 19:55:26.985826 3793 namespace.go:98] [default-tenant] adding namespace I0521 19:55:26.986446 3793 namespace.go:102] [build-service] adding namespace took 761.049µs I0521 19:55:26.987038 3793 namespace.go:102] [integration-service] adding namespace took 1.272937ms I0521 19:55:26.987704 3793 namespace.go:102] [default-tenant] adding namespace took 1.86119ms I0521 19:55:27.044552 3793 namespace.go:142] [image-controller] updating namespace I0521 19:55:27.067507 3793 namespace.go:142] [image-controller] updating namespace I0521 19:55:27.103513 3793 namespace.go:142] [build-service] updating namespace I0521 19:55:27.138028 3793 namespace.go:142] [integration-service] updating namespace I0521 19:55:27.141359 3793 namespace.go:142] [build-service] updating namespace I0521 19:55:27.176814 3793 namespace.go:142] [integration-service] updating namespace I0521 19:55:27.188318 3793 namespace.go:142] [default-tenant] updating namespace I0521 19:55:27.215238 3793 namespace.go:142] [build-service] updating namespace I0521 19:55:27.232794 3793 namespace.go:142] [default-tenant] updating namespace I0521 19:55:27.264774 3793 namespace.go:142] [integration-service] updating namespace I0521 19:55:27.297512 3793 namespace.go:98] [release-service] adding namespace I0521 19:55:27.298489 3793 namespace.go:102] [release-service] adding namespace took 960.708µs I0521 19:55:27.350749 3793 namespace.go:142] [konflux-ui] updating namespace I0521 19:55:27.365760 3793 namespace.go:142] [release-service] updating namespace I0521 19:55:27.379667 3793 namespace.go:142] [release-service] updating namespace I0521 19:55:27.392905 3793 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-49.ec2.internal I0521 19:55:27.393024 3793 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0521 19:55:27.424943 3793 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 31.944486ms I0521 19:55:27.424968 3793 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 31.971689ms I0521 19:55:27.426504 3793 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 33.621083ms, libovsdb time 1.053986ms I0521 19:55:27.491898 3793 namespace.go:98] [konflux-info] adding namespace I0521 19:55:27.492683 3793 namespace.go:102] [konflux-info] adding namespace took 768.332µs I0521 19:55:27.545589 3793 namespace.go:98] [namespace-lister] adding namespace I0521 19:55:27.546311 3793 namespace.go:102] [namespace-lister] adding namespace took 703.684µs I0521 19:55:27.593598 3793 namespace.go:142] [konflux-info] updating namespace I0521 19:55:27.642535 3793 namespace.go:142] [konflux-info] updating namespace I0521 19:55:27.670416 3793 namespace.go:98] [enterprise-contract-service] adding namespace I0521 19:55:27.671193 3793 namespace.go:102] [enterprise-contract-service] adding namespace took 759.629µs I0521 19:55:27.694240 3793 namespace.go:142] [image-controller] updating namespace I0521 19:55:27.708393 3793 namespace.go:142] [namespace-lister] updating namespace I0521 19:55:27.775398 3793 cni.go:452] [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key ] I0521 19:55:27.785003 3793 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: 15fc06bb7c17c36, network: default, NAD default, SandboxID: "15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef", PCI device ID: , UID: "ec68a5bd-02c3-4047-9437-47f6ac7ffc2a", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0521 19:55:27.819679 3793 namespace.go:142] [namespace-lister] updating namespace I0521 19:55:27.841105 3793 cni.go:473] [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:eb:85:fd:d2:c8\",\"name\":\"15fc06bb7c17c36\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d921d20-9482-4fcc-8592-bea035124383\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:55:27.854793 3793 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:55:27.894933 3793 namespace.go:142] [build-service] updating namespace I0521 19:55:27.899590 3793 namespace.go:142] [default-tenant] updating namespace I0521 19:55:27.971886 3793 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:55:28.017466 3793 namespace.go:98] [segment-bridge] adding namespace I0521 19:55:28.018352 3793 namespace.go:102] [segment-bridge] adding namespace took 866.884µs I0521 19:55:28.197555 3793 namespace.go:142] [default-tenant] updating namespace I0521 19:55:28.747711 3793 namespace.go:142] [segment-bridge] updating namespace I0521 19:55:28.747823 3793 namespace.go:142] [image-controller] updating namespace I0521 19:55:28.747963 3793 namespace.go:98] [konflux-cli] adding namespace I0521 19:55:28.748935 3793 namespace.go:102] [konflux-cli] adding namespace took 947.94µs W0521 19:55:28.809033 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0521 19:55:28.829669 3793 namespace.go:142] [build-service] updating namespace I0521 19:55:28.838578 3793 namespace.go:142] [default-tenant] updating namespace I0521 19:55:28.838748 3793 namespace.go:142] [image-controller] updating namespace I0521 19:55:28.929564 3793 namespace.go:142] [integration-service] updating namespace I0521 19:55:28.929793 3793 namespace.go:142] [release-service] updating namespace I0521 19:55:28.969784 3793 namespace.go:142] [build-service] updating namespace I0521 19:55:28.992241 3793 namespace.go:142] [default-tenant] updating namespace I0521 19:55:29.026015 3793 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-696b855848-rtzkg] creating logical port namespace-lister_namespace-lister-696b855848-rtzkg for pod on switch ip-10-0-2-49.ec2.internal I0521 19:55:29.026155 3793 kube.go:256] Updating pod namespace-lister/namespace-lister-696b855848-rtzkg I0521 19:55:29.113826 3793 namespace.go:142] [image-controller] updating namespace I0521 19:55:29.115618 3793 namespace.go:142] [segment-bridge] updating namespace I0521 19:55:29.143508 3793 pod.go:62] [namespace-lister/namespace-lister-696b855848-rtzkg] pod update took 117.389663ms I0521 19:55:29.143531 3793 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-696b855848-rtzkg] addLogicalPort annotation time took 117.418401ms I0521 19:55:29.144989 3793 pods.go:271] [namespace-lister/namespace-lister-696b855848-rtzkg] addLogicalPort took 119.001147ms, libovsdb time 991.13µs I0521 19:55:29.148625 3793 namespace.go:142] [integration-service] updating namespace I0521 19:55:29.201605 3793 namespace.go:142] [release-service] updating namespace I0521 19:55:29.243583 3793 namespace.go:142] [konflux-cli] updating namespace I0521 19:55:29.270766 3793 namespace.go:142] [release-service] updating namespace I0521 19:55:29.348833 3793 namespace.go:142] [integration-service] updating namespace I0521 19:55:29.411688 3793 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0521 19:55:29.412417 3793 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0521 19:55:29.412438 3793 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0521 19:55:29.416099 3793 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0521 19:55:29.472218 3793 namespace.go:142] [konflux-cli] updating namespace I0521 19:55:29.588460 3793 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0521 19:55:29.588492 3793 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0521 19:55:29.589690 3793 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0521 19:55:29.985082 3793 cni.go:452] [namespace-lister/namespace-lister-696b855848-rtzkg ee3d2ad0820158b0ed0a538b52db2d5755b237107f64685c01f3624ca3f99a90 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-696b855848-rtzkg ee3d2ad0820158b0ed0a538b52db2d5755b237107f64685c01f3624ca3f99a90 network default NAD default NAD key ] I0521 19:55:29.995136 3793 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-696b855848-rtzkg, hostIfaceName: ee3d2ad0820158b, network: default, NAD default, SandboxID: "ee3d2ad0820158b0ed0a538b52db2d5755b237107f64685c01f3624ca3f99a90", PCI device ID: , UID: "fc129282-d23b-45d3-94d8-c1bf11c2ff4a", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0521 19:55:30.050679 3793 cni.go:473] [namespace-lister/namespace-lister-696b855848-rtzkg ee3d2ad0820158b0ed0a538b52db2d5755b237107f64685c01f3624ca3f99a90 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-696b855848-rtzkg ee3d2ad0820158b0ed0a538b52db2d5755b237107f64685c01f3624ca3f99a90 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:b7:45:6e:cc:91\",\"name\":\"ee3d2ad0820158b\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b17f3f1-4bfb-4b6a-87fc-93e5cda29b90\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 19:55:30.599714 3793 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:55:30.792885 3793 base_network_controller_pods.go:487] [default/konflux-ui/proxy-6fbbb9b85b-w4nf8] creating logical port konflux-ui_proxy-6fbbb9b85b-w4nf8 for pod on switch ip-10-0-2-49.ec2.internal I0521 19:55:30.793056 3793 kube.go:256] Updating pod konflux-ui/proxy-6fbbb9b85b-w4nf8 I0521 19:55:30.956928 3793 namespace.go:142] [konflux-cli] updating namespace I0521 19:55:31.066525 3793 pod.go:62] [konflux-ui/proxy-6fbbb9b85b-w4nf8] pod update took 273.521375ms I0521 19:55:31.066547 3793 base_network_controller_pods.go:951] [default/konflux-ui/proxy-6fbbb9b85b-w4nf8] addLogicalPort annotation time took 273.550304ms I0521 19:55:31.067813 3793 pods.go:271] [konflux-ui/proxy-6fbbb9b85b-w4nf8] addLogicalPort took 274.94921ms, libovsdb time 870.365µs I0521 19:55:31.149920 3793 namespace.go:142] [konflux-info] updating namespace I0521 19:55:31.250662 3793 namespace.go:142] [namespace-lister] updating namespace I0521 19:55:31.458627 3793 namespace.go:142] [segment-bridge] updating namespace I0521 19:55:31.458680 3793 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:55:31.563833 3793 namespace.go:142] [konflux-cli] updating namespace I0521 19:55:31.563929 3793 namespace.go:142] [konflux-info] updating namespace I0521 19:55:31.563967 3793 namespace.go:142] [namespace-lister] updating namespace I0521 19:55:31.564132 3793 namespace.go:142] [segment-bridge] updating namespace I0521 19:55:31.646486 3793 namespace.go:142] [enterprise-contract-service] updating namespace I0521 19:55:31.684669 3793 namespace.go:142] [konflux-cli] updating namespace I0521 19:55:31.822447 3793 namespace.go:142] [konflux-info] updating namespace I0521 19:55:31.911578 3793 namespace.go:142] [namespace-lister] updating namespace I0521 19:55:31.963718 3793 namespace.go:142] [segment-bridge] updating namespace W0521 19:55:32.594942 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0521 19:55:33.363631 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 W0521 19:55:33.538653 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0521 19:55:34.356458 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 W0521 19:55:35.244133 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0521 19:55:37.257895 3793 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0521 19:55:38.612891 3793 cni.go:452] [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key ] I0521 19:55:38.622510 3793 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-6fbbb9b85b-w4nf8, hostIfaceName: 19d288c9e0f41b8, network: default, NAD default, SandboxID: "19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549", PCI device ID: , UID: "5feb424d-8ffc-4891-9091-efa919f61595", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0521 19:55:38.676605 3793 cni.go:473] [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:b7:ff:80:72:51\",\"name\":\"19d288c9e0f41b8\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9536ee5c-e9aa-4d84-89c1-9f3282649199\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0521 19:55:43.322462 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0521 19:55:43.325084 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0521 19:55:43.351738 3793 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0521 19:55:43.382811 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0521 19:55:43.382831 3793 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0521 19:55:43.410102 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0521 19:55:43.412742 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0521 19:55:43.564826 3793 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0521 19:55:43.564843 3793 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0521 19:55:45.395255 3793 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0521 19:55:45.395355 3793 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service I0521 19:55:54.704649 3793 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k6z8j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:55:54.705215 3793 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k6z8j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:55:55.405147 3793 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k6z8j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:55:55.405172 3793 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k6z8j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:55:55.413456 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-k6z8j I0521 19:55:55.413486 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-k6z8j I0521 19:56:17.336152 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 W0521 19:56:24.432328 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0521 19:56:24.432352 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0521 19:56:24.432383 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0521 19:56:24.432388 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0521 19:56:24.651858 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0521 19:56:24.651877 3793 util.go:959] Failed to build node endpoints for node ip-10-0-2-49.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0521 19:56:33.247870 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=83 I0521 19:56:49.247819 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=70 I0521 19:56:50.352267 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0521 19:56:54.225755 3793 cni.go:452] [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key ] I0521 19:56:54.308794 3793 cni.go:473] [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-6fbbb9b85b-w4nf8 19d288c9e0f41b86e86b410eb894eeeccd6626a1eb3f0550cecf48b374e2a549 network default NAD default NAD key default], result "{}", err I0521 19:56:54.372025 3793 obj_retry.go:498] Detected object konflux-ui/proxy-6fbbb9b85b-w4nf8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:56:54.372063 3793 pods.go:176] Deleting pod: konflux-ui/proxy-6fbbb9b85b-w4nf8 I0521 19:56:54.373401 3793 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-6fbbb9b85b-w4nf8, ips: 10.128.0.56 I0521 19:56:54.373430 3793 obj_retry.go:498] Detected object konflux-ui/proxy-6fbbb9b85b-w4nf8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:56:54.975122 3793 obj_retry.go:492] Detected object konflux-ui/proxy-6fbbb9b85b-w4nf8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:56:54.975146 3793 obj_retry.go:492] Detected object konflux-ui/proxy-6fbbb9b85b-w4nf8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:56:54.981947 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-6fbbb9b85b-w4nf8 I0521 19:56:54.981972 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-6fbbb9b85b-w4nf8 I0521 19:56:59.004248 3793 cni.go:452] [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key ] DEL starting CNI request [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key ] I0521 19:56:59.086943 3793 cni.go:473] [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key default] DEL finished CNI request [konflux-ui/test-tsf-konflux 15fc06bb7c17c36a8c06b346a296d5a397508813d4b201ea34b9cee7ea21e4ef network default NAD default NAD key default], result "{}", err I0521 19:56:59.137865 3793 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 19:56:59.137900 3793 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0521 19:56:59.139195 3793 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.54 I0521 19:56:59.139226 3793 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 19:56:59.156411 3793 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:56:59.156437 3793 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 19:56:59.162995 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0521 19:56:59.163020 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0521 19:57:12.251727 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=162 I0521 19:57:18.162807 3793 namespace.go:98] [managed-tenant-m5ei4] adding namespace I0521 19:57:18.163824 3793 namespace.go:102] [managed-tenant-m5ei4] adding namespace took 990.869µs I0521 19:57:18.186666 3793 namespace.go:142] [managed-tenant-m5ei4] updating namespace I0521 19:57:18.206791 3793 namespace.go:142] [managed-tenant-m5ei4] updating namespace I0521 19:57:19.227925 3793 namespace.go:142] [managed-tenant-m5ei4] updating namespace I0521 19:57:19.477004 3793 namespace.go:142] [managed-tenant-m5ei4] updating namespace I0521 19:57:19.634991 3793 namespace.go:142] [managed-tenant-m5ei4] updating namespace I0521 19:57:35.368487 3793 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0521 19:58:57.341293 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0521 19:59:15.257577 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=412 I0521 19:59:16.246142 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0521 20:00:00.254785 3793 base_network_controller_pods.go:487] [default/segment-bridge/segment-bridge-29656560-z4nh5] creating logical port segment-bridge_segment-bridge-29656560-z4nh5 for pod on switch ip-10-0-2-49.ec2.internal I0521 20:00:00.254900 3793 kube.go:256] Updating pod segment-bridge/segment-bridge-29656560-z4nh5 I0521 20:00:00.297465 3793 pod.go:62] [segment-bridge/segment-bridge-29656560-z4nh5] pod update took 42.586652ms I0521 20:00:00.297492 3793 base_network_controller_pods.go:951] [default/segment-bridge/segment-bridge-29656560-z4nh5] addLogicalPort annotation time took 42.618465ms I0521 20:00:00.298783 3793 pods.go:271] [segment-bridge/segment-bridge-29656560-z4nh5] addLogicalPort took 44.012795ms, libovsdb time 872.963µs I0521 20:00:00.306704 3793 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29656560-gjh5c for pod on switch ip-10-0-2-49.ec2.internal I0521 20:00:00.306813 3793 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c I0521 20:00:00.325551 3793 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c] pod update took 18.760287ms I0521 20:00:00.325579 3793 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c] addLogicalPort annotation time took 18.79556ms I0521 20:00:00.326975 3793 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c] addLogicalPort took 20.284699ms, libovsdb time 915.772µs I0521 20:00:00.618574 3793 cni.go:452] [segment-bridge/segment-bridge-29656560-z4nh5 5b9653cc76c1c873fb6a9251f63f7afdf37aee62b4d57212c2467b71c94a92cd network default NAD default NAD key ] ADD starting CNI request [segment-bridge/segment-bridge-29656560-z4nh5 5b9653cc76c1c873fb6a9251f63f7afdf37aee62b4d57212c2467b71c94a92cd network default NAD default NAD key ] I0521 20:00:00.628244 3793 helper_linux.go:556] ConfigureOVS: namespace: segment-bridge, podName: segment-bridge-29656560-z4nh5, hostIfaceName: 5b9653cc76c1c87, network: default, NAD default, SandboxID: "5b9653cc76c1c873fb6a9251f63f7afdf37aee62b4d57212c2467b71c94a92cd", PCI device ID: , UID: "5ff5133d-bf30-4d9d-8e5e-614b65fe4dc3", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0521 20:00:00.665598 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key ] I0521 20:00:00.677153 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29656560-gjh5c, hostIfaceName: d59cc9e1548a381, network: default, NAD default, SandboxID: "d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731", PCI device ID: , UID: "c52b45bd-78c4-43a4-a89b-101b0b38acc4", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0521 20:00:00.690652 3793 cni.go:473] [segment-bridge/segment-bridge-29656560-z4nh5 5b9653cc76c1c873fb6a9251f63f7afdf37aee62b4d57212c2467b71c94a92cd network default NAD default NAD key default] ADD finished CNI request [segment-bridge/segment-bridge-29656560-z4nh5 5b9653cc76c1c873fb6a9251f63f7afdf37aee62b4d57212c2467b71c94a92cd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:53:c7:64:50:27\",\"name\":\"5b9653cc76c1c87\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4dff99d-39c5-456e-966a-aadf9ed18ae8\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 20:00:00.941493 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:05:75:88:75:19\",\"name\":\"d59cc9e1548a381\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/235f7459-ab79-46ab-8473-32e37e763019\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 20:00:05.063008 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key ] I0521 20:00:05.160075 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c d59cc9e1548a3812afc2055085a6ddca707abe37531b3b4969c64582dfa07731 network default NAD default NAD key default], result "{}", err I0521 20:00:05.211029 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 20:00:05.211067 3793 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c I0521 20:00:05.212533 3793 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c, ips: 10.128.0.58 I0521 20:00:05.212558 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 20:00:06.233274 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:00:06.233303 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656560-gjh5c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:00:33.249586 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=80 I0521 20:01:44.251099 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=6 I0521 20:02:24.369837 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0521 20:02:35.350306 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0521 20:02:40.339215 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0521 20:02:43.360185 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0521 20:02:58.657069 3793 duration.go:461] Config duration recorder: measurement expired for pod/openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d I0521 20:03:10.377694 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0521 20:03:23.251668 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=25 I0521 20:03:30.254940 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0521 20:04:13.378229 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0521 20:04:24.340536 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0521 20:04:45.051459 3793 obj_retry.go:492] Detected object tsf-tpa/create-db-nksbr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:04:45.051501 3793 obj_retry.go:492] Detected object tsf-tpa/create-db-nksbr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:04:45.056554 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-nksbr I0521 20:04:45.056581 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-nksbr I0521 20:04:50.052801 3793 obj_retry.go:492] Detected object tsf-tpa/migrate-db-zr7dw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:04:50.052832 3793 obj_retry.go:492] Detected object tsf-tpa/migrate-db-zr7dw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:04:50.057691 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-zr7dw I0521 20:04:50.057725 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-zr7dw I0521 20:04:56.033530 3793 obj_retry.go:492] Detected object tsf-tpa/create-importers-9mdp2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:04:56.033558 3793 obj_retry.go:492] Detected object tsf-tpa/create-importers-9mdp2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:04:56.038210 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-9mdp2 I0521 20:04:56.038240 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-9mdp2 I0521 20:06:00.253103 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0521 20:06:25.345482 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0521 20:06:34.355897 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0521 20:07:24.371440 3793 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0521 20:07:29.261280 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=41 I0521 20:08:14.249594 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0521 20:08:48.354281 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0521 20:09:29.342486 3793 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0521 20:09:44.372998 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0521 20:09:58.363662 3793 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0521 20:10:00.222727 3793 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29656570-qlh2m for pod on switch ip-10-0-2-49.ec2.internal I0521 20:10:00.222839 3793 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m I0521 20:10:00.253837 3793 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m] pod update took 31.021741ms I0521 20:10:00.253863 3793 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m] addLogicalPort annotation time took 31.05328ms I0521 20:10:00.255170 3793 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m] addLogicalPort took 32.458347ms, libovsdb time 871.932µs I0521 20:10:00.587724 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key ] I0521 20:10:00.597513 3793 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29656570-qlh2m, hostIfaceName: 13388b5834f50a1, network: default, NAD default, SandboxID: "13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87", PCI device ID: , UID: "21b4041b-52f9-4e13-a4e6-68a5eb388035", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0521 20:10:00.653092 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:ab:31:4a:f6:72\",\"name\":\"13388b5834f50a1\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4799e9e4-f295-4fe1-b065-3867934603c9\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0521 20:10:04.453977 3793 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key ] I0521 20:10:04.539256 3793 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m 13388b5834f50a19383c86810715d69a45ca728b614b459748e60707032fde87 network default NAD default NAD key default], result "{}", err I0521 20:10:04.590135 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0521 20:10:04.590176 3793 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m I0521 20:10:04.591788 3793 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m, ips: 10.128.0.59 I0521 20:10:04.591821 3793 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0521 20:10:05.463845 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:10:05.463876 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656570-qlh2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:10:05.495938 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:10:05.495990 3793 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0521 20:10:05.504594 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d I0521 20:10:05.504616 3793 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29656540-7gz9d I0521 20:10:26.254773 3793 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=39