+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-91.ec2.internal ++ [[ -n ip-10-0-0-91.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-91.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0325 10:53:16.589340060 - disable conntrack on geneve port' I0325 10:53:16.589340060 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0325 10:53:16.634429121 - starting ovnkube-node + echo 'I0325 10:53:16.634429121 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-91.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-91.ec2.internal --init-node ip-10-0-0-91.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-91.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0325 10:53:16.693376 3561 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0325 10:53:16.693425 3561 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-4e8bc40fe0.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0325 10:53:16.695380 3561 kube.go:419] Waiting for certificate I0325 10:53:16.695420 3561 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0325 10:53:16.695468 3561 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0325 10:53:16.696849 3561 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0325 10:53:16.731499 3561 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0325 10:53:16.731526 3561 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0325 10:53:16.735104 3561 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0325 10:53:16.735198 3561 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-bqzh5" I0325 10:53:16.741534 3561 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-bqzh5" I0325 10:53:16.741624 3561 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0325 10:53:17.696127 3561 kube.go:426] Certificate found I0325 10:53:17.696708 3561 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0325 10:53:17.697734 3561 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0325 10:53:17.751008 3561 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-26 10:48:16 +0000 UTC" deadline="2026-03-26 07:07:53.58061067 +0000 UTC" I0325 10:53:17.751120 3561 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h14m35.829509169s" I0325 10:53:17.773371 3561 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0325 10:53:17.781535 3561 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0325 10:53:17.783437 3561 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0325 10:53:17.783569 3561 factory.go:531] Starting watch factory I0325 10:53:17.783690 3561 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783735 3561 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783809 3561 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783839 3561 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783866 3561 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783823 3561 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.784195 3561 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783854 3561 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783748 3561 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.784957 3561 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.783744 3561 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.785371 3561 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.793346 3561 controller_manager.go:368] Starting the ovnkube controller I0325 10:53:17.793362 3561 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-91.ec2.internal I0325 10:53:17.793425 3561 controller_manager.go:393] NBDB zone sync took: 54.232µs I0325 10:53:17.793475 3561 factory.go:531] Starting watch factory I0325 10:53:17.811013 3561 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.812010 3561 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.813799 3561 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.814406 3561 ovn_northd.go:121] Registering OVN northd metrics I0325 10:53:17.816608 3561 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.816978 3561 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0325 10:53:17.819739 3561 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.822149 3561 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0325 10:53:17.829204 3561 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0325 10:53:17.829231 3561 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0325 10:53:17.847479 3561 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0325 10:53:17.889601 3561 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0325 10:53:17.889636 3561 factory.go:1890] *v1.Namespace informer cache synced successfully I0325 10:53:17.889643 3561 factory.go:1890] *v1.Node informer cache synced successfully I0325 10:53:17.889646 3561 factory.go:1890] *v1.Service informer cache synced successfully I0325 10:53:17.889649 3561 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0325 10:53:17.889653 3561 factory.go:1890] *v1.Pod informer cache synced successfully I0325 10:53:17.889732 3561 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0325 10:53:17.889752 3561 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0325 10:53:17.889734 3561 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0325 10:53:17.889786 3561 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0325 10:53:17.900485 3561 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0325 10:53:17.900499 3561 factory.go:1890] *v1.Pod informer cache synced successfully I0325 10:53:17.900504 3561 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0325 10:53:17.900509 3561 factory.go:1890] *v1.Namespace informer cache synced successfully I0325 10:53:17.900514 3561 factory.go:1890] *v1.Node informer cache synced successfully I0325 10:53:17.900518 3561 factory.go:1890] *v1.Service informer cache synced successfully I0325 10:53:17.914517 3561 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0325 10:53:17.933141 3561 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0325 10:53:17.995503 3561 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0325 10:53:17.995526 3561 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0325 10:53:17.995635 3561 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:17.995652 3561 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.005953 3561 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0325 10:53:18.005971 3561 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0325 10:53:18.014487 3561 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.100960 3561 factory.go:1890] *v1.EgressIP informer cache synced successfully I0325 10:53:18.101097 3561 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.101121 3561 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.103866 3561 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.106994 3561 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0325 10:53:18.107072 3561 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.107094 3561 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.109564 3561 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.111551 3561 factory.go:1890] *v1.EgressIP informer cache synced successfully I0325 10:53:18.112696 3561 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0325 10:53:18.112778 3561 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.112793 3561 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.115181 3561 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.117170 3561 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0325 10:53:18.118306 3561 factory.go:1890] *v1.EgressService informer cache synced successfully I0325 10:53:18.118359 3561 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.118370 3561 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.122514 3561 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0325 10:53:18.127735 3561 factory.go:1890] *v1.EgressService informer cache synced successfully I0325 10:53:18.137487 3561 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.224086 3561 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0325 10:53:18.224476 3561 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0325 10:53:18.224493 3561 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0325 10:53:18.227254 3561 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0325 10:53:18.229489 3561 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0325 10:53:18.229589 3561 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.229611 3561 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.229633 3561 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.229658 3561 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.232404 3561 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.233280 3561 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0325 10:53:18.238711 3561 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0325 10:53:18.276022 3561 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0325 10:53:18.335622 3561 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0325 10:53:18.335643 3561 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0325 10:53:18.335653 3561 factory.go:636] Watch Factory start up complete, took: 552.089783ms I0325 10:53:18.335754 3561 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0325 10:53:18.344082 3561 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0325 10:53:18.344097 3561 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0325 10:53:18.344107 3561 factory.go:636] Watch Factory start up complete, took: 550.629973ms I0325 10:53:18.344117 3561 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-91.ec2.internal" zone I0325 10:53:18.392658 3561 default_node_network_controller.go:709] Initializing the default node network controller I0325 10:53:18.459686 3561 udn_isolation.go:101] Starting UDN host isolation manager I0325 10:53:18.467854 3561 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0325 10:53:18.554510 3561 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0325 10:53:18.554618 3561 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0325 10:53:18.554641 3561 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0325 10:53:18.592165 3561 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0325 10:53:18.592248 3561 default_node_network_controller.go:808] Node ip-10-0-0-91.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0325 10:53:18.598736 3561 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.91"] k8s.ovn.org/zone-name:ip-10-0-0-91.ec2.internal] on node ip-10-0-0-91.ec2.internal I0325 10:53:18.621873 3561 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0325 10:53:18.628730 3561 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0325 10:53:18.628867 3561 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0325 10:53:18.629067 3561 gateway_init.go:263] Preparing Gateway I0325 10:53:18.629081 3561 gateway_shared_intf.go:1658] Creating new gateway I0325 10:53:18.723951 3561 iptables.go:108] Creating table: filter chain: FORWARD I0325 10:53:18.731202 3561 iptables.go:108] Creating table: filter chain: OUTPUT I0325 10:53:18.738305 3561 gateway_shared_intf.go:1787] Gateway Creation Complete I0325 10:53:18.738322 3561 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-91.ec2.internal","mac-address":"12:46:b5:49:46:79","ip-addresses":["10.0.0.91/24"],"ip-address":"10.0.0.91/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:26d5b30b-bf14-4261-be1e-82538bed0af0 k8s.ovn.org/node-encap-ips:["10.0.0.91"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.91/24"} k8s.ovn.org/zone-name:ip-10-0-0-91.ec2.internal] on node ip-10-0-0-91.ec2.internal I0325 10:53:18.758888 3561 default_node_network_controller.go:518] Node connection status = connected I0325 10:53:18.844493 3561 controller_manager.go:427] Waiting for node in zone sync took: 500.361429ms I0325 10:53:18.856866 3561 controller_manager.go:608] Switching to transit router for layer2 networks I0325 10:53:18.856880 3561 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-91.ec2.internal I0325 10:53:18.887546 3561 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0325 10:53:18.893476 3561 controller_manager.go:288] SCTP support detected in OVN I0325 10:53:18.894192 3561 services_controller.go:69] Creating services controller for network=default I0325 10:53:18.894361 3561 udn_enabled_svc.go:80] Waiting for service informer to sync I0325 10:53:18.894385 3561 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0325 10:53:18.894417 3561 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0325 10:53:18.894435 3561 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0325 10:53:18.894443 3561 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0325 10:53:18.894585 3561 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0325 10:53:18.894629 3561 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0325 10:53:18.894658 3561 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 21.829µs I0325 10:53:18.894668 3561 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 41.332µs I0325 10:53:18.894677 3561 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0325 10:53:18.894688 3561 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0325 10:53:18.894699 3561 default_network_controller.go:321] Starting the default network controller I0325 10:53:18.894705 3561 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 94.661µs I0325 10:53:18.894774 3561 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 13.249µs I0325 10:53:18.895003 3561 udn_enabled_svc.go:95] Performing full resync I0325 10:53:18.895319 3561 udn_enabled_svc.go:99] Waiting for handler to sync I0325 10:53:18.895330 3561 udn_enabled_svc.go:104] Starting worker I0325 10:53:18.904219 3561 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0325 10:53:18.904287 3561 repair.go:33] Syncing exgw routes took 55.21µs I0325 10:53:18.904308 3561 default_network_controller.go:407] Starting all the Watchers... I0325 10:53:18.904461 3561 namespace.go:98] [dedicated-admin] adding namespace I0325 10:53:18.904494 3561 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0325 10:53:18.904499 3561 namespace.go:98] [openshift-console-user-settings] adding namespace I0325 10:53:18.904491 3561 namespace.go:98] [openshift-backplane-srep] adding namespace I0325 10:53:18.904506 3561 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0325 10:53:18.904523 3561 namespace.go:98] [openshift-apiserver] adding namespace I0325 10:53:18.904518 3561 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0325 10:53:18.904533 3561 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0325 10:53:18.904544 3561 namespace.go:98] [openshift-network-node-identity] adding namespace I0325 10:53:18.904544 3561 namespace.go:98] [openshift-multus] adding namespace I0325 10:53:18.904551 3561 namespace.go:98] [openshift-console-operator] adding namespace I0325 10:53:18.904551 3561 namespace.go:98] [openshift-network-console] adding namespace I0325 10:53:18.904561 3561 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0325 10:53:18.904580 3561 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0325 10:53:18.904469 3561 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0325 10:53:18.905284 3561 namespace.go:102] [dedicated-admin] adding namespace took 781.307µs I0325 10:53:18.905342 3561 namespace.go:98] [openshift-backplane-cee] adding namespace I0325 10:53:18.906004 3561 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 1.485866ms I0325 10:53:18.906025 3561 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0325 10:53:18.906710 3561 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 676.501µs I0325 10:53:18.906728 3561 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0325 10:53:18.907492 3561 namespace.go:102] [openshift-console-user-settings] adding namespace took 2.967712ms I0325 10:53:18.907513 3561 namespace.go:98] [kube-node-lease] adding namespace I0325 10:53:18.908210 3561 namespace.go:102] [openshift-backplane-srep] adding namespace took 3.668525ms I0325 10:53:18.908232 3561 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0325 10:53:18.908909 3561 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 4.38002ms I0325 10:53:18.908955 3561 namespace.go:98] [kube-public] adding namespace I0325 10:53:18.909581 3561 namespace.go:102] [openshift-apiserver] adding namespace took 5.046709ms I0325 10:53:18.909604 3561 namespace.go:98] [openshift-route-controller-manager] adding namespace I0325 10:53:18.910290 3561 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.754664ms I0325 10:53:18.910310 3561 namespace.go:98] [openshift-dns] adding namespace I0325 10:53:18.910902 3561 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.360406ms I0325 10:53:18.910921 3561 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace W0325 10:53:18.911409 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-lls2x: pod openshift-multus/network-metrics-daemon-lls2x: no pod IPs found W0325 10:53:18.911423 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-qf66d: pod openshift-multus/network-metrics-daemon-qf66d: no pod IPs found W0325 10:53:18.911431 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-t76gw: pod openshift-multus/network-metrics-daemon-t76gw: no pod IPs found I0325 10:53:18.911447 3561 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.895706ms I0325 10:53:18.911464 3561 namespace.go:98] [openshift-apiserver-operator] adding namespace I0325 10:53:18.911962 3561 namespace.go:102] [openshift-multus] adding namespace took 7.409046ms I0325 10:53:18.911977 3561 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0325 10:53:18.912467 3561 namespace.go:102] [openshift-console-operator] adding namespace took 7.909419ms I0325 10:53:18.912481 3561 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0325 10:53:18.912921 3561 namespace.go:102] [openshift-network-console] adding namespace took 8.360524ms I0325 10:53:18.912949 3561 namespace.go:98] [openshift-network-diagnostics] adding namespace I0325 10:53:18.913459 3561 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.890814ms I0325 10:53:18.913477 3561 namespace.go:98] [openshift-must-gather-operator] adding namespace I0325 10:53:18.913953 3561 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 9.360912ms I0325 10:53:18.913966 3561 namespace.go:98] [openshift-ingress-operator] adding namespace I0325 10:53:18.914450 3561 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 9.822807ms I0325 10:53:18.914468 3561 namespace.go:98] [openshift-network-operator] adding namespace I0325 10:53:18.914969 3561 namespace.go:102] [openshift-backplane-cee] adding namespace took 9.612924ms I0325 10:53:18.914988 3561 namespace.go:98] [openshift-ingress] adding namespace I0325 10:53:18.916051 3561 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 9.315821ms I0325 10:53:18.916068 3561 namespace.go:98] [openshift-authentication-operator] adding namespace I0325 10:53:18.916740 3561 namespace.go:102] [kube-node-lease] adding namespace took 9.218698ms I0325 10:53:18.916794 3561 namespace.go:98] [openshift-etcd] adding namespace I0325 10:53:18.917410 3561 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 9.168873ms I0325 10:53:18.917430 3561 namespace.go:98] [openshift-operators] adding namespace I0325 10:53:18.918140 3561 namespace.go:102] [kube-public] adding namespace took 9.174984ms I0325 10:53:18.918169 3561 namespace.go:98] [openshift] adding namespace I0325 10:53:18.918789 3561 namespace.go:102] [openshift-route-controller-manager] adding namespace took 9.177632ms I0325 10:53:18.918808 3561 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0325 10:53:18.919610 3561 namespace.go:102] [openshift-dns] adding namespace took 9.291128ms I0325 10:53:18.919639 3561 namespace.go:98] [openshift-ingress-canary] adding namespace I0325 10:53:18.920189 3561 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 9.260456ms I0325 10:53:18.920207 3561 namespace.go:98] [openshift-logging] adding namespace I0325 10:53:18.920875 3561 namespace.go:102] [openshift-apiserver-operator] adding namespace took 9.404033ms I0325 10:53:18.920891 3561 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0325 10:53:18.921569 3561 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 9.583363ms I0325 10:53:18.921590 3561 namespace.go:98] [openshift-machine-api] adding namespace W0325 10:53:18.922151 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-5dkd6: pod openshift-network-diagnostics/network-check-target-5dkd6: no pod IPs found W0325 10:53:18.922166 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-hhc6d: pod openshift-network-diagnostics/network-check-target-hhc6d: no pod IPs found W0325 10:53:18.922177 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-tnw6p: pod openshift-network-diagnostics/network-check-target-tnw6p: no pod IPs found I0325 10:53:18.922191 3561 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 9.704183ms I0325 10:53:18.922204 3561 namespace.go:98] [openshift-dns-operator] adding namespace I0325 10:53:18.922884 3561 namespace.go:102] [openshift-network-diagnostics] adding namespace took 9.927666ms I0325 10:53:18.922905 3561 namespace.go:98] [openshift-infra] adding namespace I0325 10:53:18.923559 3561 namespace.go:102] [openshift-must-gather-operator] adding namespace took 10.073891ms I0325 10:53:18.923578 3561 namespace.go:98] [openshift-backplane-tam] adding namespace I0325 10:53:18.924258 3561 namespace.go:102] [openshift-ingress-operator] adding namespace took 10.282211ms I0325 10:53:18.924279 3561 namespace.go:98] [openshift-insights] adding namespace I0325 10:53:18.924854 3561 namespace.go:102] [openshift-network-operator] adding namespace took 10.378427ms I0325 10:53:18.924872 3561 namespace.go:98] [openshift-console] adding namespace I0325 10:53:18.925568 3561 namespace.go:102] [openshift-ingress] adding namespace took 10.571218ms I0325 10:53:18.925588 3561 namespace.go:98] [openshift-node] adding namespace I0325 10:53:18.926342 3561 namespace.go:102] [openshift-authentication-operator] adding namespace took 10.266044ms I0325 10:53:18.926362 3561 namespace.go:98] [openshift-kube-apiserver] adding namespace I0325 10:53:18.926945 3561 namespace.go:102] [openshift-etcd] adding namespace took 10.122293ms I0325 10:53:18.927007 3561 namespace.go:98] [open-cluster-management-2p8mq20h6bsbq55jr1en15t0ro8fskh6] adding namespace I0325 10:53:18.927599 3561 namespace.go:102] [openshift-operators] adding namespace took 10.160029ms I0325 10:53:18.927616 3561 namespace.go:98] [openshift-kube-scheduler] adding namespace I0325 10:53:18.928293 3561 namespace.go:102] [openshift] adding namespace took 10.114402ms I0325 10:53:18.928336 3561 namespace.go:98] [openshift-config] adding namespace I0325 10:53:18.928952 3561 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 10.11771ms I0325 10:53:18.928981 3561 namespace.go:98] [kube-system] adding namespace I0325 10:53:18.929587 3561 namespace.go:102] [openshift-ingress-canary] adding namespace took 9.940727ms I0325 10:53:18.929603 3561 namespace.go:98] [openshift-authentication] adding namespace I0325 10:53:18.930314 3561 namespace.go:102] [openshift-logging] adding namespace took 10.100324ms I0325 10:53:18.930331 3561 namespace.go:98] [openshift-config-managed] adding namespace I0325 10:53:18.931034 3561 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 10.135216ms I0325 10:53:18.931049 3561 namespace.go:98] [openshift-controller-manager] adding namespace I0325 10:53:18.931722 3561 namespace.go:102] [openshift-machine-api] adding namespace took 10.122742ms I0325 10:53:18.931775 3561 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0325 10:53:18.932375 3561 namespace.go:102] [openshift-dns-operator] adding namespace took 10.162826ms I0325 10:53:18.932393 3561 namespace.go:98] [openshift-host-network] adding namespace I0325 10:53:18.933081 3561 namespace.go:102] [openshift-infra] adding namespace took 10.16651ms I0325 10:53:18.933098 3561 namespace.go:98] [openshift-monitoring] adding namespace I0325 10:53:18.933773 3561 namespace.go:102] [openshift-backplane-tam] adding namespace took 10.186628ms I0325 10:53:18.933792 3561 namespace.go:98] [default] adding namespace I0325 10:53:18.934394 3561 namespace.go:102] [openshift-insights] adding namespace took 10.106761ms I0325 10:53:18.934413 3561 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0325 10:53:18.935109 3561 namespace.go:102] [openshift-console] adding namespace took 10.229388ms I0325 10:53:18.935130 3561 namespace.go:98] [openshift-marketplace] adding namespace I0325 10:53:18.935651 3561 namespace.go:102] [openshift-node] adding namespace took 10.056783ms I0325 10:53:18.935668 3561 namespace.go:98] [openshift-image-registry] adding namespace I0325 10:53:18.936298 3561 namespace.go:102] [openshift-kube-apiserver] adding namespace took 9.919137ms I0325 10:53:18.936876 3561 namespace.go:102] [open-cluster-management-2p8mq20h6bsbq55jr1en15t0ro8fskh6] adding namespace took 9.856666ms I0325 10:53:18.936890 3561 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0325 10:53:18.937526 3561 namespace.go:102] [openshift-kube-scheduler] adding namespace took 9.901485ms I0325 10:53:18.937548 3561 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace W0325 10:53:18.938185 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-qlhc8: pod kube-system/global-pull-secret-syncer-qlhc8: no pod IPs found W0325 10:53:18.938197 3561 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-x99cj: pod kube-system/global-pull-secret-syncer-x99cj: no pod IPs found I0325 10:53:18.938232 3561 namespace.go:102] [openshift-config] adding namespace took 9.882227ms I0325 10:53:18.938252 3561 namespace.go:98] [openshift-operators-redhat] adding namespace I0325 10:53:18.938921 3561 namespace.go:102] [kube-system] adding namespace took 9.932076ms I0325 10:53:18.938950 3561 namespace.go:98] [openshift-package-operator] adding namespace I0325 10:53:18.939563 3561 namespace.go:102] [openshift-authentication] adding namespace took 9.953886ms I0325 10:53:18.939611 3561 namespace.go:98] [openshift-machine-config-operator] adding namespace I0325 10:53:18.940410 3561 namespace.go:102] [openshift-config-managed] adding namespace took 10.070356ms I0325 10:53:18.940437 3561 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0325 10:53:18.941140 3561 namespace.go:102] [openshift-controller-manager] adding namespace took 10.084219ms I0325 10:53:18.941157 3561 namespace.go:98] [openshift-backplane-cse] adding namespace I0325 10:53:18.941676 3561 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 9.889245ms I0325 10:53:18.941692 3561 namespace.go:98] [openshift-cluster-version] adding namespace I0325 10:53:18.942398 3561 namespace.go:102] [openshift-host-network] adding namespace took 9.997925ms I0325 10:53:18.942414 3561 namespace.go:98] [openshift-backplane-mobb] adding namespace I0325 10:53:18.943010 3561 namespace.go:102] [openshift-monitoring] adding namespace took 9.904939ms I0325 10:53:18.943030 3561 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0325 10:53:18.943658 3561 namespace.go:102] [default] adding namespace took 9.859471ms I0325 10:53:18.943675 3561 namespace.go:98] [openshift-backplane] adding namespace I0325 10:53:18.944338 3561 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 9.917794ms I0325 10:53:18.944968 3561 namespace.go:102] [openshift-marketplace] adding namespace took 9.829569ms I0325 10:53:18.944986 3561 namespace.go:98] [openshift-backplane-csm] adding namespace I0325 10:53:18.945604 3561 namespace.go:102] [openshift-image-registry] adding namespace took 9.928234ms I0325 10:53:18.946250 3561 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 9.353612ms I0325 10:53:18.946271 3561 namespace.go:98] [openshift-config-operator] adding namespace I0325 10:53:18.946904 3561 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 9.347849ms I0325 10:53:18.946925 3561 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0325 10:53:18.947498 3561 namespace.go:102] [openshift-operators-redhat] adding namespace took 9.237887ms I0325 10:53:18.948230 3561 namespace.go:102] [openshift-package-operator] adding namespace took 9.272176ms I0325 10:53:18.948250 3561 namespace.go:98] [openshift-service-ca-operator] adding namespace I0325 10:53:18.948841 3561 namespace.go:102] [openshift-machine-config-operator] adding namespace took 9.21901ms I0325 10:53:18.948872 3561 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0325 10:53:18.949423 3561 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.976399ms I0325 10:53:18.950112 3561 namespace.go:102] [openshift-backplane-cse] adding namespace took 8.946745ms I0325 10:53:18.950593 3561 namespace.go:102] [openshift-cluster-version] adding namespace took 8.89405ms I0325 10:53:18.951260 3561 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.839315ms I0325 10:53:18.951972 3561 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.931939ms I0325 10:53:18.952595 3561 namespace.go:102] [openshift-backplane] adding namespace took 8.912391ms I0325 10:53:18.953262 3561 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.268116ms I0325 10:53:18.953283 3561 namespace.go:98] [openshift-customer-monitoring] adding namespace I0325 10:53:18.953963 3561 namespace.go:102] [openshift-config-operator] adding namespace took 7.684191ms I0325 10:53:18.954548 3561 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.615596ms I0325 10:53:18.955267 3561 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.009681ms I0325 10:53:18.956561 3561 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.682108ms I0325 10:53:18.957218 3561 namespace.go:102] [openshift-customer-monitoring] adding namespace took 3.92684ms I0325 10:53:18.957733 3561 zone_ic_handler.go:186] Time taken to create transit switch: 400.585µs I0325 10:53:18.957822 3561 master.go:532] Adding or Updating local node "ip-10-0-0-91.ec2.internal" for network "default" I0325 10:53:18.957909 3561 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-111.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-111.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-111.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-111.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-111.ec2.internal I0325 10:53:18.958012 3561 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-56.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-56.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-56.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-56.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-56.ec2.internal W0325 10:53:18.968355 3561 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-qlhc8: pod kube-system/global-pull-secret-syncer-qlhc8: no pod IPs found W0325 10:53:18.968380 3561 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-qf66d: pod openshift-multus/network-metrics-daemon-qf66d: no pod IPs found W0325 10:53:18.968392 3561 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-hhc6d: pod openshift-network-diagnostics/network-check-target-hhc6d: no pod IPs found I0325 10:53:18.969520 3561 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-91.ec2.internal for the network default I0325 10:53:18.970581 3561 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0325 10:53:18.970623 3561 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0325 10:53:18.970643 3561 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0325 10:53:18.970645 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-91.ec2.internal in network "default" I0325 10:53:18.970720 3561 node_tracker.go:169] Node ip-10-0-0-91.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:18.970735 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:18.970747 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-56.ec2.internal in network "default" I0325 10:53:18.970760 3561 node_tracker.go:235] Node ip-10-0-1-56.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-56.ec2.internal" I0325 10:53:18.970773 3561 node_tracker.go:169] Node ip-10-0-1-56.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:18.970779 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:18.970794 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0325 10:53:18.970816 3561 node_tracker.go:235] Node ip-10-0-2-111.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-111.ec2.internal" I0325 10:53:18.970828 3561 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:18.970835 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:19.071771 3561 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0325 10:53:19.071798 3561 services_controller.go:208] Setting up event handlers for services for network=default I0325 10:53:19.071891 3561 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0325 10:53:19.071960 3561 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0325 10:53:19.071983 3561 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0325 10:53:19.173096 3561 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0325 10:53:19.173122 3561 repair.go:60] Starting repairing loop for services I0325 10:53:19.173219 3561 repair.go:131] Deleted 0 stale service LBs I0325 10:53:19.173246 3561 repair.go:137] Deleted 0 stale Chassis Template Vars I0325 10:53:19.173283 3561 repair.go:62] Finished repairing loop for services: 163.164µs I0325 10:53:19.173337 3561 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0325 10:53:19.173350 3561 services_controller.go:254] Starting workers for network=default I0325 10:53:19.173759 3561 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-lls2x, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-lls2x: suppressed error logged: pod openshift-multus/network-metrics-daemon-lls2x: no pod IPs found I0325 10:53:19.173775 3561 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-x99cj, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-x99cj: suppressed error logged: pod kube-system/global-pull-secret-syncer-x99cj: no pod IPs found I0325 10:53:19.173925 3561 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-qlhc8] creating logical port kube-system_global-pull-secret-syncer-qlhc8 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:19.173784 3561 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-t76gw, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-t76gw: suppressed error logged: pod openshift-multus/network-metrics-daemon-t76gw: no pod IPs found I0325 10:53:19.173956 3561 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-qf66d] creating logical port openshift-multus_network-metrics-daemon-qf66d for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:19.173977 3561 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-hhc6d] creating logical port openshift-network-diagnostics_network-check-target-hhc6d for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:19.173864 3561 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-tnw6p, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-tnw6p: suppressed error logged: pod openshift-network-diagnostics/network-check-target-tnw6p: no pod IPs found I0325 10:53:19.174103 3561 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-qf66d I0325 10:53:19.174120 3561 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-qlhc8 I0325 10:53:19.174179 3561 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-hhc6d I0325 10:53:19.191816 3561 pod.go:62] [openshift-network-diagnostics/network-check-target-hhc6d] pod update took 17.775394ms I0325 10:53:19.191847 3561 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-hhc6d] addLogicalPort annotation time took 17.807035ms I0325 10:53:19.192011 3561 pod.go:62] [kube-system/global-pull-secret-syncer-qlhc8] pod update took 17.998698ms I0325 10:53:19.192078 3561 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-qlhc8] addLogicalPort annotation time took 18.067835ms I0325 10:53:19.192999 3561 pods.go:271] [openshift-network-diagnostics/network-check-target-hhc6d] addLogicalPort took 19.028431ms, libovsdb time 876.31µs I0325 10:53:19.193475 3561 pods.go:271] [kube-system/global-pull-secret-syncer-qlhc8] addLogicalPort took 19.556433ms, libovsdb time 1.110453ms I0325 10:53:19.197868 3561 pod.go:62] [openshift-multus/network-metrics-daemon-qf66d] pod update took 23.849262ms I0325 10:53:19.197887 3561 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-qf66d] addLogicalPort annotation time took 23.869096ms I0325 10:53:19.198820 3561 pods.go:271] [openshift-multus/network-metrics-daemon-qf66d] addLogicalPort took 24.892424ms, libovsdb time 705.873µs I0325 10:53:19.198860 3561 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-5dkd6, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5dkd6: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5dkd6: no pod IPs found I0325 10:53:19.199305 3561 admin_network_policy_controller.go:227] Starting controller default-network-controller I0325 10:53:19.199336 3561 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0325 10:53:19.199349 3561 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0325 10:53:19.199355 3561 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0325 10:53:19.199573 3561 repair.go:29] Repairing admin network policies took 210.645µs I0325 10:53:19.199751 3561 repair.go:92] Repairing baseline admin network policies took 164.018µs I0325 10:53:19.200321 3561 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0325 10:53:19.200349 3561 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0325 10:53:19.203044 3561 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0325 10:53:19.203064 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0325 10:53:19.203082 3561 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0325 10:53:19.205166 3561 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0325 10:53:19.205196 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0325 10:53:19.205210 3561 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0325 10:53:19.205799 3561 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0325 10:53:19.205819 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0325 10:53:19.205848 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0325 10:53:19.207870 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0325 10:53:19.207906 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0325 10:53:19.207919 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0325 10:53:19.208539 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0325 10:53:19.208558 3561 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0325 10:53:19.208567 3561 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0325 10:53:19.211511 3561 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0325 10:53:19.211534 3561 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0325 10:53:19.211544 3561 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0325 10:53:19.212600 3561 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0325 10:53:19.212619 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0325 10:53:19.213068 3561 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0325 10:53:19.213082 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0325 10:53:19.214041 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0325 10:53:19.214064 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0325 10:53:19.214097 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0325 10:53:19.216082 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0325 10:53:19.216101 3561 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0325 10:53:19.216110 3561 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0325 10:53:19.218989 3561 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0325 10:53:19.219011 3561 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0325 10:53:19.219020 3561 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0325 10:53:19.220774 3561 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0325 10:53:19.220796 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0325 10:53:19.220805 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0325 10:53:19.221518 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0325 10:53:19.221537 3561 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0325 10:53:19.221547 3561 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0325 10:53:19.222714 3561 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0325 10:53:19.222739 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0325 10:53:19.222750 3561 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0325 10:53:19.223380 3561 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0325 10:53:19.223401 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0325 10:53:19.223420 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0325 10:53:19.224038 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0325 10:53:19.224063 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0325 10:53:19.224074 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0325 10:53:19.225631 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0325 10:53:19.225656 3561 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0325 10:53:19.225665 3561 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0325 10:53:19.226097 3561 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0325 10:53:19.226119 3561 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0325 10:53:19.226130 3561 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0325 10:53:19.226500 3561 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0325 10:53:19.226522 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0325 10:53:19.226533 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0325 10:53:19.227173 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0325 10:53:19.227210 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0325 10:53:19.227222 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0325 10:53:19.227841 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0325 10:53:19.227863 3561 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0325 10:53:19.227873 3561 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0325 10:53:19.228501 3561 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0325 10:53:19.228520 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0325 10:53:19.228528 3561 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0325 10:53:19.228962 3561 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0325 10:53:19.228983 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0325 10:53:19.228996 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0325 10:53:19.229674 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0325 10:53:19.229695 3561 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0325 10:53:19.229709 3561 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0325 10:53:19.231088 3561 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0325 10:53:19.231112 3561 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0325 10:53:19.231126 3561 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0325 10:53:19.232542 3561 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0325 10:53:19.232566 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0325 10:53:19.232576 3561 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0325 10:53:19.233198 3561 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0325 10:53:19.233220 3561 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0325 10:53:19.233227 3561 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0325 10:53:19.233595 3561 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0325 10:53:19.233649 3561 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0325 10:53:19.233668 3561 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0325 10:53:19.234330 3561 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0325 10:53:19.234352 3561 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0325 10:53:19.234377 3561 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0325 10:53:19.235657 3561 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0325 10:53:19.241797 3561 egressfirewall.go:319] Starting EgressFirewall controller I0325 10:53:19.241839 3561 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0325 10:53:19.241870 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0325 10:53:19.241877 3561 controller.go:133] Adding controller egress-firewall-controller event handlers I0325 10:53:19.241889 3561 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0325 10:53:19.241906 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0325 10:53:19.241917 3561 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0325 10:53:19.242279 3561 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0325 10:53:19.242298 3561 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0325 10:53:19.242311 3561 egressqos.go:193] Setting up event handlers for EgressQoS I0325 10:53:19.242391 3561 egressfirewall.go:628] Syncing node "ip-10-0-0-91.ec2.internal" for egress firewall I0325 10:53:19.242418 3561 egressfirewall.go:628] Syncing node "ip-10-0-1-56.ec2.internal" for egress firewall I0325 10:53:19.242426 3561 egressfirewall.go:628] Syncing node "ip-10-0-2-111.ec2.internal" for egress firewall I0325 10:53:19.242548 3561 egressqos.go:245] Starting EgressQoS Controller I0325 10:53:19.242565 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0325 10:53:19.242576 3561 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0325 10:53:19.242588 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0325 10:53:19.242601 3561 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0325 10:53:19.242618 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0325 10:53:19.242628 3561 shared_informer.go:356] "Caches are synced" controller="egressqos" I0325 10:53:19.242637 3561 egressqos.go:259] Repairing EgressQoSes I0325 10:53:19.242643 3561 egressqos.go:399] Starting repairing loop for egressqos I0325 10:53:19.242771 3561 egressqos.go:401] Finished repairing loop for egressqos: 128.515µs I0325 10:53:19.242798 3561 egressservice_zone.go:132] Setting up event handlers for Egress Services I0325 10:53:19.242839 3561 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-91.ec2.internal I0325 10:53:19.242920 3561 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-91.ec2.internal : 82.711µs I0325 10:53:19.243189 3561 egressservice_zone.go:210] Starting Egress Services Controller I0325 10:53:19.243242 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0325 10:53:19.243258 3561 shared_informer.go:356] "Caches are synced" controller="egressservices" I0325 10:53:19.243267 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0325 10:53:19.243276 3561 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0325 10:53:19.243285 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0325 10:53:19.243293 3561 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0325 10:53:19.243300 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0325 10:53:19.243306 3561 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0325 10:53:19.243311 3561 egressservice_zone.go:228] Repairing Egress Services I0325 10:53:19.243756 3561 master_controller.go:88] Starting Admin Policy Based Route Controller I0325 10:53:19.243795 3561 external_controller.go:278] Starting Admin Policy Based Route Controller I0325 10:53:19.243836 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-91.ec2.internal I0325 10:53:19.243854 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-91.ec2.internal: 20.02µs I0325 10:53:19.243864 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-56.ec2.internal I0325 10:53:19.243869 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-56.ec2.internal: 6.091µs I0325 10:53:19.243875 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0325 10:53:19.243879 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 4.663µs I0325 10:53:19.244142 3561 default_network_controller.go:568] Completing all the Watchers took 339.823403ms I0325 10:53:19.244158 3561 default_network_controller.go:572] Starting unidling controllers I0325 10:53:19.244171 3561 unidle.go:46] Registering OVN SB ControllerEvent handler I0325 10:53:19.244179 3561 unidle.go:63] Populating Initial ContollerEvent events I0325 10:53:19.244194 3561 unidle.go:79] Setting up event handlers for services I0325 10:53:19.244300 3561 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0325 10:53:19.260015 3561 default_node_network_controller.go:518] Node connection status = connected I0325 10:53:19.760335 3561 default_node_network_controller.go:518] Node connection status = connected I0325 10:53:19.870344 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0325 10:53:19.870433 3561 default_network_controller.go:1008] Node "ip-10-0-2-111.ec2.internal" in remote zone "ip-10-0-2-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0325 10:53:19.870440 3561 node_tracker.go:235] Node ip-10-0-2-111.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-111.ec2.internal" I0325 10:53:19.870505 3561 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:19.870527 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:20.010006 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0325 10:53:20.010073 3561 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:20.010089 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:20.010094 3561 default_network_controller.go:1008] Node "ip-10-0-2-111.ec2.internal" in remote zone "ip-10-0-2-111.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0325 10:53:20.010993 3561 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-111.ec2.internal for the network default I0325 10:53:20.012484 3561 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-111.ec2.internal" on network "default" took: 1.516318ms I0325 10:53:20.260898 3561 default_node_network_controller.go:518] Node connection status = connected I0325 10:53:20.554783 3561 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0325 10:53:20.759497 3561 default_node_network_controller.go:518] Node connection status = connected I0325 10:53:20.774677 3561 bridgeconfig.go:537] Gateway is ready I0325 10:53:20.774698 3561 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0325 10:53:20.775366 3561 egressfirewall.go:628] Syncing node "ip-10-0-1-56.ec2.internal" for egress firewall I0325 10:53:20.775393 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-56.ec2.internal I0325 10:53:20.775455 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-56.ec2.internal: 63.031µs I0325 10:53:20.775407 3561 default_network_controller.go:1008] Node "ip-10-0-1-56.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0325 10:53:20.784883 3561 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.91. Updating OVN encap IP. I0325 10:53:20.789653 3561 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.91 - it is already configured I0325 10:53:20.789670 3561 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.91/24:{} 172.20.0.1/32:{}]. Updating annotations. I0325 10:53:20.789921 3561 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.91/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-91.ec2.internal","mac-address":"12:46:b5:49:46:79","ip-addresses":["10.0.0.91/24"],"ip-address":"10.0.0.91/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:26d5b30b-bf14-4261-be1e-82538bed0af0 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.91/24"}] on node ip-10-0-0-91.ec2.internal I0325 10:53:20.805697 3561 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.91 I0325 10:53:20.805789 3561 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-91.ec2.internal I0325 10:53:20.806359 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-91.ec2.internal I0325 10:53:20.806421 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-91.ec2.internal: 65.463µs I0325 10:53:20.806385 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-91.ec2.internal in network "default" I0325 10:53:20.806525 3561 node_tracker.go:169] Node ip-10-0-0-91.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:20.806549 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:20.806400 3561 egressfirewall.go:628] Syncing node "ip-10-0-0-91.ec2.internal" for egress firewall I0325 10:53:20.806445 3561 master.go:532] Adding or Updating local node "ip-10-0-0-91.ec2.internal" for network "default" I0325 10:53:20.811883 3561 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-91.ec2.internal. Updating no re-route policies I0325 10:53:20.811949 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-gsf4m I0325 10:53:20.812002 3561 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-91.ec2.internal I0325 10:53:20.812038 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-tnw6p I0325 10:53:20.812052 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-tnw6p I0325 10:53:20.812053 3561 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-91.ec2.internal I0325 10:53:20.812010 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-p6whq I0325 10:53:20.812095 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-p6whq I0325 10:53:20.811952 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-hztzq I0325 10:53:20.812098 3561 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-tnw6p, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-tnw6p: suppressed error logged: pod openshift-network-diagnostics/network-check-target-tnw6p: no pod IPs found I0325 10:53:20.812106 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-hztzq I0325 10:53:20.811964 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-t76gw I0325 10:53:20.812117 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-t76gw I0325 10:53:20.812119 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-hztzq after 0 failed attempt(s) I0325 10:53:20.812120 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-p6whq after 0 failed attempt(s) I0325 10:53:20.812095 3561 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-91.ec2.internal after 0 failed attempt(s) I0325 10:53:20.811964 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-fc6s5 I0325 10:53:20.812141 3561 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-t76gw, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-t76gw: suppressed error logged: pod openshift-multus/network-metrics-daemon-t76gw: no pod IPs found I0325 10:53:20.812143 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-fc6s5 I0325 10:53:20.811979 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-hhc6d I0325 10:53:20.812179 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-hhc6d I0325 10:53:20.812216 3561 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-hhc6d] creating logical port openshift-network-diagnostics_network-check-target-hhc6d for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:20.811992 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kkqpz I0325 10:53:20.812260 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kkqpz I0325 10:53:20.812277 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kkqpz after 0 failed attempt(s) I0325 10:53:20.811992 3561 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-qlhc8 I0325 10:53:20.812287 3561 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-qlhc8 I0325 10:53:20.812310 3561 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-qlhc8] creating logical port kube-system_global-pull-secret-syncer-qlhc8 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:20.811999 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-r5ls4 I0325 10:53:20.812326 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-r5ls4 I0325 10:53:20.812345 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-r5ls4 after 0 failed attempt(s) I0325 10:53:20.812012 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-91.ec2.internal I0325 10:53:20.812371 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-91.ec2.internal I0325 10:53:20.812448 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-91.ec2.internal after 0 failed attempt(s) I0325 10:53:20.812019 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-5h9jq I0325 10:53:20.812504 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-5h9jq I0325 10:53:20.812525 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-5h9jq after 0 failed attempt(s) I0325 10:53:20.812056 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-gsf4m I0325 10:53:20.812555 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-gsf4m after 0 failed attempt(s) I0325 10:53:20.811948 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-vw2fj I0325 10:53:20.812572 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-vw2fj I0325 10:53:20.812592 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-vw2fj after 0 failed attempt(s) I0325 10:53:20.811963 3561 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-qpvzg I0325 10:53:20.812614 3561 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-qpvzg I0325 10:53:20.812636 3561 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-qpvzg after 0 failed attempt(s) I0325 10:53:20.811974 3561 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-qf66d I0325 10:53:20.812655 3561 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-qf66d I0325 10:53:20.812682 3561 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-qf66d] creating logical port openshift-multus_network-metrics-daemon-qf66d for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:20.812163 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-fc6s5 after 0 failed attempt(s) I0325 10:53:20.811988 3561 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-9cws5 I0325 10:53:20.812927 3561 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-9cws5 I0325 10:53:20.813002 3561 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-9cws5 after 0 failed attempt(s) I0325 10:53:20.813163 3561 pods.go:271] [openshift-network-diagnostics/network-check-target-hhc6d] addLogicalPort took 955.687µs, libovsdb time 357.922µs I0325 10:53:20.813176 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-hhc6d after 0 failed attempt(s) I0325 10:53:20.813329 3561 pods.go:271] [kube-system/global-pull-secret-syncer-qlhc8] addLogicalPort took 1.024086ms, libovsdb time 438.817µs I0325 10:53:20.813346 3561 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-qlhc8 after 0 failed attempt(s) I0325 10:53:20.813732 3561 pods.go:271] [openshift-multus/network-metrics-daemon-qf66d] addLogicalPort took 1.056024ms, libovsdb time 540.721µs I0325 10:53:20.813749 3561 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-qf66d after 0 failed attempt(s) I0325 10:53:20.820855 3561 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0325 10:53:20.831737 3561 iptables.go:108] Creating table: mangle chain: OUTPUT I0325 10:53:20.834900 3561 iptables.go:108] Creating table: nat chain: OUTPUT I0325 10:53:20.858899 3561 iptables.go:108] Creating table: nat chain: PREROUTING I0325 10:53:20.913991 3561 iptables.go:108] Creating table: filter chain: FORWARD I0325 10:53:20.930121 3561 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0325 10:53:20.930209 3561 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0325 10:53:20.930249 3561 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0325 10:53:20.930336 3561 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0325 10:53:20.930383 3561 gateway_init.go:325] Gateway and management port readiness took 2.174597006s I0325 10:53:20.930409 3561 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0325 10:53:20.930442 3561 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0325 10:53:20.930454 3561 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0325 10:53:20.930462 3561 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0325 10:53:20.930516 3561 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0325 10:53:20.930585 3561 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 46.705µs I0325 10:53:20.930773 3561 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0325 10:53:20.930795 3561 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 12.83µs I0325 10:53:20.930803 3561 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 30.937µs I0325 10:53:20.930809 3561 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0325 10:53:20.930818 3561 nad_controller.go:173] [node-nad-controller NAD controller]: started I0325 10:53:20.930826 3561 default_node_network_controller.go:898] Starting the default node network controller I0325 10:53:20.930831 3561 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0325 10:53:20.930841 3561 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 11.501µs I0325 10:53:20.930975 3561 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0325 10:53:20.930983 3561 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0325 10:53:20.931012 3561 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b26780 0xc0002c44d0 0xc0002c4540 0xc0002c4620 0xc0002c4690 0xc0002c4770 0xc0002c47e0 0xc000b0d180 0xc0002c4930 0xc0002c49a0 0xc000b0c370 0xc006147180 0xc0002c4a10 map[0x2f67040:0xc0049dc000 0x2f67580:0xc0049dc180 0x2f68000:0xc002a93860 0x2f682a0:0xc002a938c0 0x2f687e0:0xc0049dc060 0x2f68a80:0xc0049dc0c0 0x2f68d20:0xc0049dc120 0x2f69500:0xc0049dc1e0 0x2f697a0:0xc0049dc240 0x2f93d40:0xc001901260 0x2f940e0:0xc0019012c0 0x2f94bc0:0xc000b629c0 0x2f96520:0xc000ba7500 0x2f97000:0xc002a93800 0x2f97e80:0xc0009c80c0] 0xc0000c3340 0} I0325 10:53:20.931066 3561 gateway.go:151] Starting gateway service sync I0325 10:53:20.931481 3561 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0325 10:53:20.933962 3561 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0325 10:53:20.955667 3561 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0325 10:53:20.956738 3561 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0325 10:53:20.958021 3561 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0325 10:53:20.992640 3561 gateway.go:168] Gateway service sync done. Time taken: 61.561812ms I0325 10:53:20.993259 3561 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31356 I0325 10:53:20.993429 3561 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b26780 0xc0002c44d0 0xc0002c4540 0xc0002c4620 0xc0002c4690 0xc0002c4770 0xc0002c47e0 0xc000b0d180 0xc0002c4930 0xc0002c49a0 0xc000b0c370 0xc006147180 0xc0002c4a10 map[0x2f67040:0xc0049dc000 0x2f67580:0xc0049dc180 0x2f68000:0xc002a93860 0x2f682a0:0xc002a938c0 0x2f687e0:0xc0049dc060 0x2f68a80:0xc0049dc0c0 0x2f68d20:0xc0049dc120 0x2f69500:0xc0049dc1e0 0x2f697a0:0xc0049dc240 0x2f93d40:0xc001901260 0x2f940e0:0xc0019012c0 0x2f94bc0:0xc000b629c0 0x2f96520:0xc000ba7500 0x2f97000:0xc002a93800 0x2f97e80:0xc0009c80c0] 0xc0000c3340 0} I0325 10:53:20.993877 3561 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b26780 0xc0002c44d0 0xc0002c4540 0xc0002c4620 0xc0002c4690 0xc0002c4770 0xc0002c47e0 0xc000b0d180 0xc0002c4930 0xc0002c49a0 0xc000b0c370 0xc006147180 0xc0002c4a10 map[0x2f67040:0xc0049dc000 0x2f67580:0xc0049dc180 0x2f68000:0xc002a93860 0x2f682a0:0xc002a938c0 0x2f687e0:0xc0049dc060 0x2f68a80:0xc0049dc0c0 0x2f68d20:0xc0049dc120 0x2f69500:0xc0049dc1e0 0x2f697a0:0xc0049dc240 0x2f93d40:0xc001901260 0x2f940e0:0xc0019012c0 0x2f94bc0:0xc000b629c0 0x2f96520:0xc000ba7500 0x2f97000:0xc002a93800 0x2f97e80:0xc0009c80c0] 0xc0000c3340 0} I0325 10:53:20.993911 3561 gateway.go:519] Reconciling gateway with updates I0325 10:53:20.999735 3561 gateway.go:326] Spawning Conntrack Rule Check Thread I0325 10:53:20.999817 3561 default_node_network_controller.go:943] Gateway and management port readiness took 68.826839ms I0325 10:53:20.999840 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0325 10:53:20.999908 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0325 10:53:20.999997 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0325 10:53:21.000057 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0325 10:53:21.000112 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0325 10:53:21.000143 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0325 10:53:21.000180 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0325 10:53:21.000213 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0325 10:53:21.000225 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0325 10:53:21.000240 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0325 10:53:20.999909 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0325 10:53:21.000254 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0325 10:53:21.000260 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0325 10:53:21.000147 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0325 10:53:21.000272 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0325 10:53:21.000115 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0325 10:53:21.000277 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0325 10:53:21.000285 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0325 10:53:21.000334 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0325 10:53:21.000081 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0325 10:53:20.999859 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0325 10:53:21.000417 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0325 10:53:21.000467 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0325 10:53:20.999863 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0325 10:53:20.999895 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0325 10:53:20.999862 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0325 10:53:21.000490 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0325 10:53:21.000495 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0325 10:53:21.000525 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0325 10:53:20.999866 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0325 10:53:21.000553 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0325 10:53:21.000566 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0325 10:53:21.000482 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0325 10:53:20.999872 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0325 10:53:21.000629 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0325 10:53:21.000638 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0325 10:53:21.000647 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0325 10:53:20.999879 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0325 10:53:21.000661 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0325 10:53:20.999878 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0325 10:53:21.000739 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0325 10:53:21.000747 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0325 10:53:20.999881 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0325 10:53:21.000757 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0325 10:53:21.000761 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0325 10:53:20.999886 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0325 10:53:21.000851 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0325 10:53:20.999888 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0325 10:53:21.000871 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0325 10:53:20.999882 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0325 10:53:21.001021 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0325 10:53:21.001036 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0325 10:53:20.999897 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0325 10:53:21.001053 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0325 10:53:21.001064 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0325 10:53:20.999898 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0325 10:53:21.001080 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0325 10:53:20.999897 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0325 10:53:21.001108 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0325 10:53:20.999900 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0325 10:53:21.001168 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0325 10:53:20.999911 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0325 10:53:21.001225 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0325 10:53:20.999915 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0325 10:53:20.999843 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0325 10:53:21.001315 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0325 10:53:20.999915 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0325 10:53:21.001369 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0325 10:53:20.999920 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0325 10:53:21.001413 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0325 10:53:20.999920 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0325 10:53:21.001453 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0325 10:53:20.999946 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0325 10:53:21.001506 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0325 10:53:20.999956 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0325 10:53:21.001542 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0325 10:53:20.999959 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0325 10:53:21.001580 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0325 10:53:20.999955 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0325 10:53:21.001616 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0325 10:53:21.001624 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0325 10:53:20.999974 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0325 10:53:21.001637 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0325 10:53:21.000084 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0325 10:53:21.001676 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0325 10:53:21.001683 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0325 10:53:21.000090 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0325 10:53:21.001697 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0325 10:53:21.000093 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0325 10:53:21.001731 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0325 10:53:21.001745 3561 port_claim.go:71] Svc openshift-ingress/router-default: port 32017 is already open, no action needed I0325 10:53:21.001754 3561 port_claim.go:71] Svc openshift-ingress/router-default: port 31610 is already open, no action needed I0325 10:53:21.000102 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0325 10:53:21.001803 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0325 10:53:21.000106 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0325 10:53:21.001853 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0325 10:53:21.000118 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0325 10:53:21.000123 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0325 10:53:21.001920 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0325 10:53:21.001928 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0325 10:53:21.000129 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0325 10:53:21.001960 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0325 10:53:21.000135 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0325 10:53:21.002001 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0325 10:53:21.000368 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0325 10:53:21.002023 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0325 10:53:20.999871 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0325 10:53:21.002070 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0325 10:53:21.000613 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0325 10:53:21.001203 3561 node_ip_handler_linux.go:160] Node IP manager is running W0325 10:53:21.036358 3561 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0325 10:53:21.036780 3561 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0325 10:53:21.036920 3561 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0325 10:53:21.047046 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0325 10:53:21.047078 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0325 10:53:21.047090 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0325 10:53:21.047100 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0325 10:53:21.047109 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0325 10:53:21.047117 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0325 10:53:21.047125 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0325 10:53:21.047133 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0325 10:53:21.047143 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0325 10:53:21.047155 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0325 10:53:21.047166 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0325 10:53:21.047174 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0325 10:53:21.047198 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0325 10:53:21.047227 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0325 10:53:21.047237 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0325 10:53:21.088640 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0325 10:53:21.088668 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0325 10:53:21.108455 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0325 10:53:21.108468 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0325 10:53:21.108485 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0325 10:53:21.108493 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0325 10:53:21.108499 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0325 10:53:21.108504 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0325 10:53:21.108510 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0325 10:53:21.108519 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0325 10:53:22.558835 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0325 10:53:22.558894 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-111.ec2.internal in network "default" I0325 10:53:22.558899 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 70.536µs I0325 10:53:22.558904 3561 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-111.ec2.internal. Updating no re-route policies I0325 10:53:22.558964 3561 node_tracker.go:169] Node ip-10-0-2-111.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:22.558894 3561 egressfirewall.go:628] Syncing node "ip-10-0-2-111.ec2.internal" for egress firewall I0325 10:53:22.558990 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:22.755554 3561 default_node_network_controller.go:1364] Starting node controller node sync W0325 10:53:22.755615 3561 egressip_healthcheck.go:76] Health checking using insecure connection I0325 10:53:22.755676 3561 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0325 10:53:22.767327 3561 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.754457ms I0325 10:53:22.767438 3561 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-111.ec2.internal", IP: 10.0.2.111 to PMTUD blocking rules I0325 10:53:22.767447 3561 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-111.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0325 10:53:22.773267 3561 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0325 10:53:22.773521 3561 egressservice_node.go:93] Setting up event handlers for Egress Services I0325 10:53:22.773626 3561 egressservice_node.go:183] Starting Egress Services Controller I0325 10:53:22.773639 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0325 10:53:22.773647 3561 shared_informer.go:356] "Caches are synced" controller="egressservices" I0325 10:53:22.773652 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0325 10:53:22.773657 3561 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0325 10:53:22.773662 3561 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0325 10:53:22.773666 3561 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0325 10:53:22.773670 3561 egressservice_node.go:197] Repairing Egress Services I0325 10:53:22.783113 3561 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0325 10:53:22.783128 3561 external_controller.go:278] Starting Admin Policy Based Route Controller I0325 10:53:22.783274 3561 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0325 10:53:22.783287 3561 link_network_manager.go:117] Link manager is running I0325 10:53:22.783299 3561 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0325 10:53:22.783347 3561 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0325 10:53:22.783618 3561 vrf_manager.go:128] VRF manager is running I0325 10:53:22.783736 3561 node_controller_manager.go:437] Removing flows to drop GARP I0325 10:53:22.783748 3561 gateway.go:519] Reconciling gateway with updates I0325 10:53:22.802791 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0325 10:53:22.802804 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0325 10:53:22.802809 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0325 10:53:22.802818 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0325 10:53:22.802826 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0325 10:53:22.802830 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0325 10:53:22.802831 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0325 10:53:22.802832 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0325 10:53:22.802838 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0325 10:53:22.802843 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0325 10:53:22.802850 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0325 10:53:22.802853 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0325 10:53:22.802859 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0325 10:53:22.802861 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0325 10:53:22.802820 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0325 10:53:22.802872 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0325 10:53:22.802849 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0325 10:53:22.802806 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0325 10:53:22.803022 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0325 10:53:22.803049 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0325 10:53:22.803057 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0325 10:53:22.803061 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0325 10:53:22.802862 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0325 10:53:22.803076 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0325 10:53:22.803024 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0325 10:53:22.803118 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0325 10:53:22.803120 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0325 10:53:22.803134 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0325 10:53:22.803169 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0325 10:53:22.802840 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0325 10:53:22.803237 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0325 10:53:22.802809 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0325 10:53:22.803252 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0325 10:53:22.802791 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0325 10:53:22.803260 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0325 10:53:22.803295 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0325 10:53:22.802820 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0325 10:53:22.803308 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0325 10:53:22.803328 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0325 10:53:22.802839 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0325 10:53:22.802904 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0325 10:53:22.803343 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0325 10:53:22.803381 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0325 10:53:22.802912 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0325 10:53:22.802913 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0325 10:53:22.803395 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0325 10:53:22.803399 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0325 10:53:22.802825 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0325 10:53:22.803408 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0325 10:53:22.803444 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0325 10:53:22.802920 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0325 10:53:22.802922 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0325 10:53:22.803458 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0325 10:53:22.803483 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0325 10:53:22.802950 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0325 10:53:22.803498 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0325 10:53:22.802953 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0325 10:53:22.803540 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0325 10:53:22.802960 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0325 10:53:22.803598 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0325 10:53:22.802963 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0325 10:53:22.803635 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0325 10:53:22.802964 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0325 10:53:22.802968 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0325 10:53:22.803733 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0325 10:53:22.803741 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0325 10:53:22.802968 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0325 10:53:22.802971 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0325 10:53:22.803754 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0325 10:53:22.802972 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0325 10:53:22.803803 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0325 10:53:22.802972 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0325 10:53:22.802978 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0325 10:53:22.803858 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0325 10:53:22.802981 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0325 10:53:22.803902 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0325 10:53:22.803910 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0325 10:53:22.802984 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0325 10:53:22.803921 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0325 10:53:22.803928 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0325 10:53:22.802985 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0325 10:53:22.803981 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0325 10:53:22.802985 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0325 10:53:22.804040 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0325 10:53:22.802991 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0325 10:53:22.804071 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0325 10:53:22.802992 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0325 10:53:22.804103 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0325 10:53:22.802993 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0325 10:53:22.804147 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0325 10:53:22.804152 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0325 10:53:22.802997 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0325 10:53:22.804163 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0325 10:53:22.802999 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0325 10:53:22.804187 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0325 10:53:22.803003 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0325 10:53:22.804276 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0325 10:53:22.803004 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0325 10:53:22.804308 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0325 10:53:22.804313 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0325 10:53:22.803006 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0325 10:53:22.804323 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0325 10:53:22.804334 3561 port_claim.go:71] Svc openshift-ingress/router-default: port 32017 is already open, no action needed I0325 10:53:22.804345 3561 port_claim.go:71] Svc openshift-ingress/router-default: port 31610 is already open, no action needed I0325 10:53:22.803014 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0325 10:53:22.804388 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0325 10:53:22.803014 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0325 10:53:22.803013 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0325 10:53:22.804514 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0325 10:53:22.803014 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0325 10:53:22.804562 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0325 10:53:22.803015 3561 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0325 10:53:22.804634 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0325 10:53:22.803031 3561 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0325 10:53:22.822773 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0325 10:53:22.822845 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0325 10:53:22.822861 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0325 10:53:22.822868 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0325 10:53:22.859590 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0325 10:53:22.859619 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0325 10:53:22.859627 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0325 10:53:22.859635 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0325 10:53:22.859643 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0325 10:53:22.859651 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0325 10:53:22.859664 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0325 10:53:22.859680 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0325 10:53:22.859687 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0325 10:53:22.859692 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0325 10:53:22.897324 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0325 10:53:22.897353 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0325 10:53:22.897369 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0325 10:53:22.897377 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0325 10:53:22.897388 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0325 10:53:22.897395 3561 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0325 10:53:24.735957 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-91.ec2.internal I0325 10:53:24.736057 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-91.ec2.internal: 107.123µs I0325 10:53:24.773338 3561 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5d466cd54f-g75n8] creating logical port openshift-image-registry_image-registry-5d466cd54f-g75n8 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.773527 3561 kube.go:256] Updating pod openshift-image-registry/image-registry-5d466cd54f-g75n8 I0325 10:53:24.799982 3561 pod.go:62] [openshift-image-registry/image-registry-5d466cd54f-g75n8] pod update took 26.538506ms I0325 10:53:24.800013 3561 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-5d466cd54f-g75n8] addLogicalPort annotation time took 26.573564ms I0325 10:53:24.801630 3561 pods.go:271] [openshift-image-registry/image-registry-5d466cd54f-g75n8] addLogicalPort took 28.309165ms, libovsdb time 1.11889ms I0325 10:53:24.815139 3561 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-44hsd] creating logical port openshift-dns_dns-default-44hsd for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.815264 3561 kube.go:256] Updating pod openshift-dns/dns-default-44hsd I0325 10:53:24.826962 3561 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-9cws5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:53:24.826996 3561 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-9cws5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:53:24.842142 3561 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-q75z5] creating logical port openshift-insights_insights-operator-5577457d66-q75z5 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.842148 3561 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-lcxfd for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.842348 3561 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-q75z5 I0325 10:53:24.842416 3561 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd I0325 10:53:24.845960 3561 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-bd8d6ccd8-tmbbk] creating logical port openshift-ingress_router-default-bd8d6ccd8-tmbbk for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.845970 3561 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-llgk8] creating logical port openshift-console-operator_console-operator-8777bd9bb-llgk8 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.846097 3561 kube.go:256] Updating pod openshift-ingress/router-default-bd8d6ccd8-tmbbk I0325 10:53:24.846212 3561 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-llgk8 I0325 10:53:24.846249 3561 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6b966944cc-nrfcl] creating logical port openshift-ingress_router-default-6b966944cc-nrfcl for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.846286 3561 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-dmk9b for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.846573 3561 kube.go:256] Updating pod openshift-ingress/router-default-6b966944cc-nrfcl I0325 10:53:24.846660 3561 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b I0325 10:53:24.856059 3561 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-bm6cx for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.856135 3561 pod.go:62] [openshift-dns/dns-default-44hsd] pod update took 40.898446ms I0325 10:53:24.856161 3561 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-44hsd] addLogicalPort annotation time took 40.927148ms I0325 10:53:24.856237 3561 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx I0325 10:53:24.857558 3561 pods.go:271] [openshift-dns/dns-default-44hsd] addLogicalPort took 42.435591ms, libovsdb time 1.076726ms I0325 10:53:24.857642 3561 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-b8jfn for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.857773 3561 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn I0325 10:53:24.947046 3561 pod.go:62] [openshift-insights/insights-operator-5577457d66-q75z5] pod update took 104.731701ms I0325 10:53:24.947078 3561 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-q75z5] addLogicalPort annotation time took 104.770302ms I0325 10:53:24.948985 3561 pods.go:271] [openshift-insights/insights-operator-5577457d66-q75z5] addLogicalPort took 106.861512ms, libovsdb time 1.550688ms I0325 10:53:24.956132 3561 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-llgk8] pod update took 109.940944ms I0325 10:53:24.956344 3561 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-llgk8] addLogicalPort annotation time took 110.154124ms I0325 10:53:24.956399 3561 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-pkkmd for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.956534 3561 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd I0325 10:53:24.956569 3561 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-9v6q7 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.956651 3561 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7 I0325 10:53:24.956251 3561 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd] pod update took 113.848358ms I0325 10:53:24.956735 3561 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd] addLogicalPort annotation time took 114.337225ms I0325 10:53:24.956843 3561 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.956976 3561 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg I0325 10:53:24.957118 3561 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx] pod update took 100.902094ms I0325 10:53:24.957136 3561 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx] addLogicalPort annotation time took 100.922497ms I0325 10:53:24.957972 3561 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-llgk8] addLogicalPort took 112.011693ms, libovsdb time 1.292721ms I0325 10:53:24.958486 3561 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29573930-vn68s for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.958649 3561 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s I0325 10:53:24.958729 3561 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-7q2dv for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:24.958991 3561 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv I0325 10:53:24.959163 3561 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx] addLogicalPort took 103.130847ms, libovsdb time 1.732802ms I0325 10:53:24.959180 3561 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd] addLogicalPort took 117.04721ms, libovsdb time 2.10431ms I0325 10:53:24.972958 3561 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn] pod update took 115.207793ms I0325 10:53:24.972979 3561 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn] addLogicalPort annotation time took 115.229749ms I0325 10:53:24.973103 3561 pod.go:62] [openshift-ingress/router-default-bd8d6ccd8-tmbbk] pod update took 127.032612ms I0325 10:53:24.973121 3561 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-bd8d6ccd8-tmbbk] addLogicalPort annotation time took 127.058468ms I0325 10:53:24.974167 3561 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn] addLogicalPort took 116.530925ms, libovsdb time 873.206µs I0325 10:53:24.974473 3561 pods.go:271] [openshift-ingress/router-default-bd8d6ccd8-tmbbk] addLogicalPort took 128.540469ms, libovsdb time 1.0888ms I0325 10:53:25.020801 3561 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b] pod update took 174.161438ms I0325 10:53:25.020824 3561 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b] addLogicalPort annotation time took 174.18536ms I0325 10:53:25.021920 3561 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b] addLogicalPort took 175.641768ms, libovsdb time 722.22µs I0325 10:53:25.033751 3561 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7] pod update took 77.117172ms I0325 10:53:25.033769 3561 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7] addLogicalPort annotation time took 77.139424ms I0325 10:53:25.034800 3561 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7] addLogicalPort took 78.237524ms, libovsdb time 663.452µs I0325 10:53:25.034845 3561 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-8df4t] creating logical port openshift-ingress-canary_ingress-canary-8df4t for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:25.034907 3561 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-8df4t I0325 10:53:25.042088 3561 pod.go:62] [openshift-ingress/router-default-6b966944cc-nrfcl] pod update took 195.531599ms I0325 10:53:25.042107 3561 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-6b966944cc-nrfcl] addLogicalPort annotation time took 195.553495ms I0325 10:53:25.043141 3561 pods.go:271] [openshift-ingress/router-default-6b966944cc-nrfcl] addLogicalPort took 196.899797ms, libovsdb time 757.834µs I0325 10:53:25.049310 3561 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s] pod update took 90.681839ms I0325 10:53:25.049366 3561 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv] pod update took 90.403686ms I0325 10:53:25.049429 3561 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv] addLogicalPort annotation time took 90.465047ms I0325 10:53:25.049376 3561 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s] addLogicalPort annotation time took 90.746982ms I0325 10:53:25.049388 3561 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg] pod update took 92.454963ms I0325 10:53:25.049819 3561 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg] addLogicalPort annotation time took 92.885679ms I0325 10:53:25.050545 3561 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv] addLogicalPort took 91.823998ms, libovsdb time 782.789µs I0325 10:53:25.050998 3561 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s] addLogicalPort took 92.521929ms, libovsdb time 932.759µs I0325 10:53:25.051469 3561 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg] addLogicalPort took 94.636059ms, libovsdb time 1.288322ms I0325 10:53:25.051572 3561 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-rk9cb for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:25.051687 3561 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb I0325 10:53:25.082554 3561 pod.go:62] [openshift-ingress-canary/ingress-canary-8df4t] pod update took 47.659382ms I0325 10:53:25.082576 3561 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-8df4t] addLogicalPort annotation time took 47.687464ms I0325 10:53:25.083870 3561 pods.go:271] [openshift-ingress-canary/ingress-canary-8df4t] addLogicalPort took 49.026686ms, libovsdb time 880.022µs I0325 10:53:25.139743 3561 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb] pod update took 88.08166ms I0325 10:53:25.139765 3561 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb] addLogicalPort annotation time took 88.111462ms I0325 10:53:25.141044 3561 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb] addLogicalPort took 89.488119ms, libovsdb time 896.956µs I0325 10:53:25.222275 3561 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-llgk8 e9c399e8a48945d50ca9830541055cbe3588e3b8024249d298e3a60ae9a5c631 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-llgk8 e9c399e8a48945d50ca9830541055cbe3588e3b8024249d298e3a60ae9a5c631 network default NAD default] I0325 10:53:25.233915 3561 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd] pod update took 277.399219ms I0325 10:53:25.233978 3561 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd] addLogicalPort annotation time took 277.476396ms I0325 10:53:25.236165 3561 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd] addLogicalPort took 279.775465ms, libovsdb time 1.275838ms I0325 10:53:25.241176 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-llgk8, hostIfaceName: e9c399e8a48945d, network: default, NAD default, SandboxID: "e9c399e8a48945d50ca9830541055cbe3588e3b8024249d298e3a60ae9a5c631", PCI device ID: , UID: "71f611f2-e87e-4350-bbb8-efb7a8050931", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0325 10:53:25.286642 3561 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-llgk8 e9c399e8a48945d50ca9830541055cbe3588e3b8024249d298e3a60ae9a5c631 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-llgk8 e9c399e8a48945d50ca9830541055cbe3588e3b8024249d298e3a60ae9a5c631 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:ff:1e:d0:d4:31\",\"name\":\"e9c399e8a48945d\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/73a518fc-6a87-40c7-96a0-d97973c1670c\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.322781 3561 cni.go:403] [openshift-ingress-canary/ingress-canary-8df4t 20c8d2c124ae2ce5c706e99146670eba3fd9c8e3959f93b7b657906cc335ada3 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-8df4t 20c8d2c124ae2ce5c706e99146670eba3fd9c8e3959f93b7b657906cc335ada3 network default NAD default] I0325 10:53:25.326639 3561 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb 4d91bc821d085b9fc3848ec137774a4743b2df39b5cf659d4c00b337d73d6891 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb 4d91bc821d085b9fc3848ec137774a4743b2df39b5cf659d4c00b337d73d6891 network default NAD default] I0325 10:53:25.335704 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-8df4t, hostIfaceName: 20c8d2c124ae2ce, network: default, NAD default, SandboxID: "20c8d2c124ae2ce5c706e99146670eba3fd9c8e3959f93b7b657906cc335ada3", PCI device ID: , UID: "ad545f45-b038-46de-bff3-c819c43cb55a", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0325 10:53:25.338864 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-rk9cb, hostIfaceName: 4d91bc821d085b9, network: default, NAD default, SandboxID: "4d91bc821d085b9fc3848ec137774a4743b2df39b5cf659d4c00b337d73d6891", PCI device ID: , UID: "1a2c87bd-1341-494f-b0e7-bf7e8856f0f3", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0325 10:53:25.372620 3561 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv c9c6a4d4ef1828afd53b1ddf9af13a10d4461a56d60bd6520a258ca42b274a2a network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv c9c6a4d4ef1828afd53b1ddf9af13a10d4461a56d60bd6520a258ca42b274a2a network default NAD default] I0325 10:53:25.376673 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default] I0325 10:53:25.413525 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29573930-vn68s, hostIfaceName: 96374716645acfe, network: default, NAD default, SandboxID: "96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949", PCI device ID: , UID: "c5af267d-3b7d-4092-809e-022b179a14e2", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0325 10:53:25.417129 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-7q2dv, hostIfaceName: c9c6a4d4ef1828a, network: default, NAD default, SandboxID: "c9c6a4d4ef1828afd53b1ddf9af13a10d4461a56d60bd6520a258ca42b274a2a", PCI device ID: , UID: "7d333539-eaf5-4680-8108-3ffb020385b0", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0325 10:53:25.417886 3561 cni.go:424] [openshift-ingress-canary/ingress-canary-8df4t 20c8d2c124ae2ce5c706e99146670eba3fd9c8e3959f93b7b657906cc335ada3 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-8df4t 20c8d2c124ae2ce5c706e99146670eba3fd9c8e3959f93b7b657906cc335ada3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:ec:bb:f7:8c:7b\",\"name\":\"20c8d2c124ae2ce\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/995066f1-12e0-46cf-beff-3aa3aa516c79\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.423659 3561 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb 4d91bc821d085b9fc3848ec137774a4743b2df39b5cf659d4c00b337d73d6891 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-rk9cb 4d91bc821d085b9fc3848ec137774a4743b2df39b5cf659d4c00b337d73d6891 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:84:e6:32:52:42\",\"name\":\"4d91bc821d085b9\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/adfc9701-7880-45ac-b6ba-ca2e707e92a3\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.465037 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:65:56:76:26:82\",\"name\":\"96374716645acfe\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc2c6f63-4e32-4081-ae39-4911364dc19a\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.472989 3561 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv c9c6a4d4ef1828afd53b1ddf9af13a10d4461a56d60bd6520a258ca42b274a2a network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-7q2dv c9c6a4d4ef1828afd53b1ddf9af13a10d4461a56d60bd6520a258ca42b274a2a network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:7b:1e:c8:20:28\",\"name\":\"c9c6a4d4ef1828a\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf667ad9-9e03-4a6d-a0d5-a9057ee0abba\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.517412 3561 cni.go:403] [openshift-insights/insights-operator-5577457d66-q75z5 ab8e07d60bf9c8433ba0dbe2edda2da2d01af999a425c8a6f75eaa6f59df1b89 network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-q75z5 ab8e07d60bf9c8433ba0dbe2edda2da2d01af999a425c8a6f75eaa6f59df1b89 network default NAD default] I0325 10:53:25.527187 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-q75z5, hostIfaceName: ab8e07d60bf9c84, network: default, NAD default, SandboxID: "ab8e07d60bf9c8433ba0dbe2edda2da2d01af999a425c8a6f75eaa6f59df1b89", PCI device ID: , UID: "e41ffd80-cdac-4571-b051-b963b6865f1a", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0325 10:53:25.544527 3561 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b 5153beac008bee8fca011964ad27f6a023a2b25dc941d8de90f79331610f82ae network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b 5153beac008bee8fca011964ad27f6a023a2b25dc941d8de90f79331610f82ae network default NAD default] I0325 10:53:25.555547 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-dmk9b, hostIfaceName: 5153beac008bee8, network: default, NAD default, SandboxID: "5153beac008bee8fca011964ad27f6a023a2b25dc941d8de90f79331610f82ae", PCI device ID: , UID: "ad256369-4844-4b96-b436-354a9f457a4b", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0325 10:53:25.575070 3561 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd 7c14e007ee061dccde912206291a8904a251ce441fbd30e4f4bd9e63e27568f6 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd 7c14e007ee061dccde912206291a8904a251ce441fbd30e4f4bd9e63e27568f6 network default NAD default] I0325 10:53:25.584891 3561 cni.go:424] [openshift-insights/insights-operator-5577457d66-q75z5 ab8e07d60bf9c8433ba0dbe2edda2da2d01af999a425c8a6f75eaa6f59df1b89 network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-q75z5 ab8e07d60bf9c8433ba0dbe2edda2da2d01af999a425c8a6f75eaa6f59df1b89 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:a6:8b:ff:a3:dd\",\"name\":\"ab8e07d60bf9c84\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36afae77-d445-4d16-89cb-ec2cf4c3a70a\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.586593 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-pkkmd, hostIfaceName: 7c14e007ee061dc, network: default, NAD default, SandboxID: "7c14e007ee061dccde912206291a8904a251ce441fbd30e4f4bd9e63e27568f6", PCI device ID: , UID: "37a6e2d6-79b6-44dd-8090-0834cc850614", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0325 10:53:25.609123 3561 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b 5153beac008bee8fca011964ad27f6a023a2b25dc941d8de90f79331610f82ae network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-dmk9b 5153beac008bee8fca011964ad27f6a023a2b25dc941d8de90f79331610f82ae network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:c9:ea:1a:dc:d6\",\"name\":\"5153beac008bee8\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/372afbe1-e7e3-4d9c-b8e3-2493e2042613\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.621571 3561 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg 518387abae20e403bfcd3ff033e0d155a5f28ed1bcc424f57865b2a4e398e084 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg 518387abae20e403bfcd3ff033e0d155a5f28ed1bcc424f57865b2a4e398e084 network default NAD default] I0325 10:53:25.630687 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg, hostIfaceName: 518387abae20e40, network: default, NAD default, SandboxID: "518387abae20e403bfcd3ff033e0d155a5f28ed1bcc424f57865b2a4e398e084", PCI device ID: , UID: "4f40a6f3-6b5f-4c7d-b3ff-9c40489bb9e4", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0325 10:53:25.640412 3561 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd 7c14e007ee061dccde912206291a8904a251ce441fbd30e4f4bd9e63e27568f6 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-pkkmd 7c14e007ee061dccde912206291a8904a251ce441fbd30e4f4bd9e63e27568f6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:78:09:b7:8f:8c\",\"name\":\"7c14e007ee061dc\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/602c1f51-ddca-4481-be7b-2cca972201e0\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.674658 3561 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg 518387abae20e403bfcd3ff033e0d155a5f28ed1bcc424f57865b2a4e398e084 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-rqpcg 518387abae20e403bfcd3ff033e0d155a5f28ed1bcc424f57865b2a4e398e084 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:59:a6:09:b0:74\",\"name\":\"518387abae20e40\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ad8401b-2192-490c-93db-10c3ff500560\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:25.830053 3561 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-9cws5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:53:25.830131 3561 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-9cws5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:53:25.842610 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-56.ec2.internal in network "default" I0325 10:53:25.842648 3561 node_tracker.go:235] Node ip-10-0-1-56.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-56.ec2.internal" I0325 10:53:25.842663 3561 node_tracker.go:169] Node ip-10-0-1-56.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:25.842659 3561 default_network_controller.go:1008] Node "ip-10-0-1-56.ec2.internal" in remote zone "ip-10-0-1-56.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0325 10:53:25.842676 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:25.887063 3561 default_network_controller.go:1008] Node "ip-10-0-1-56.ec2.internal" in remote zone "ip-10-0-1-56.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0325 10:53:25.999915 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-56.ec2.internal in network "default" I0325 10:53:26.000033 3561 default_network_controller.go:1008] Node "ip-10-0-1-56.ec2.internal" in remote zone "ip-10-0-1-56.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0325 10:53:26.000150 3561 node_tracker.go:169] Node ip-10-0-1-56.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:26.000172 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:26.000888 3561 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-56.ec2.internal for the network default I0325 10:53:26.002917 3561 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-56.ec2.internal" on network "default" took: 2.048021ms I0325 10:53:26.589668 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-111.ec2.internal I0325 10:53:26.589768 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-111.ec2.internal: 108.182µs I0325 10:53:28.551916 3561 egressfirewall.go:628] Syncing node "ip-10-0-1-56.ec2.internal" for egress firewall I0325 10:53:28.552062 3561 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-56.ec2.internal in network "default" I0325 10:53:28.552128 3561 node_tracker.go:169] Node ip-10-0-1-56.ec2.internal switch + router changed, syncing services in network "default" I0325 10:53:28.552148 3561 services_controller.go:574] Full service sync requested for network=default I0325 10:53:28.552250 3561 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-56.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0325 10:53:28.552268 3561 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-56.ec2.internal", IP: 10.0.1.56 to PMTUD blocking rules I0325 10:53:28.552887 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-56.ec2.internal I0325 10:53:28.552981 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-56.ec2.internal: 94.128µs I0325 10:53:28.560329 3561 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-56.ec2.internal. Updating no re-route policies I0325 10:53:28.573858 3561 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-56.ec2.internal", IP: 10.0.1.56 to PMTUD blocking rules I0325 10:53:28.573964 3561 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-56.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0325 10:53:32.058431 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-56.ec2.internal I0325 10:53:32.058461 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-56.ec2.internal: 38.162µs W0325 10:53:35.858382 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0325 10:53:36.000276 3561 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0325 10:53:37.922635 3561 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0325 10:53:37.923795 3561 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.13504ms I0325 10:53:37.943304 3561 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0325 10:53:38.254148 3561 namespace.go:98] [openshift-service-ca] adding namespace I0325 10:53:38.255180 3561 namespace.go:102] [openshift-service-ca] adding namespace took 932.187µs I0325 10:53:38.274748 3561 namespace.go:142] [openshift-service-ca] updating namespace I0325 10:53:39.225816 3561 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-wmtzd] creating logical port openshift-insights_insights-runtime-extractor-wmtzd for pod on switch ip-10-0-0-91.ec2.internal I0325 10:53:39.225971 3561 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-wmtzd I0325 10:53:39.241083 3561 pod.go:62] [openshift-insights/insights-runtime-extractor-wmtzd] pod update took 15.183053ms I0325 10:53:39.241108 3561 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-wmtzd] addLogicalPort annotation time took 15.212608ms I0325 10:53:39.242500 3561 pods.go:271] [openshift-insights/insights-runtime-extractor-wmtzd] addLogicalPort took 16.701941ms, libovsdb time 941.62µs W0325 10:53:47.487332 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0325 10:53:53.221348 3561 cni.go:403] [openshift-multus/network-metrics-daemon-qf66d 1c7f88d6202e4bd5da8055188ff54c9e8d5a0f8be2f110542097b5239f862054 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-qf66d 1c7f88d6202e4bd5da8055188ff54c9e8d5a0f8be2f110542097b5239f862054 network default NAD default] I0325 10:53:53.231360 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-qf66d, hostIfaceName: 1c7f88d6202e4bd, network: default, NAD default, SandboxID: "1c7f88d6202e4bd5da8055188ff54c9e8d5a0f8be2f110542097b5239f862054", PCI device ID: , UID: "06e5e64e-bd11-4f54-aaf1-ab854526f2d6", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0325 10:53:53.274827 3561 cni.go:424] [openshift-multus/network-metrics-daemon-qf66d 1c7f88d6202e4bd5da8055188ff54c9e8d5a0f8be2f110542097b5239f862054 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-qf66d 1c7f88d6202e4bd5da8055188ff54c9e8d5a0f8be2f110542097b5239f862054 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:88:96:dd:3c:58\",\"name\":\"1c7f88d6202e4bd\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/daf37050-b1d5-4759-9768-87ebdc574f1b\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:53.509269 3561 cni.go:403] [openshift-network-diagnostics/network-check-target-hhc6d afdd8dada842306ee995d2d5d3572a1e0dd8d5deff2195370db6b98f6bed9fee network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-hhc6d afdd8dada842306ee995d2d5d3572a1e0dd8d5deff2195370db6b98f6bed9fee network default NAD default] I0325 10:53:53.519223 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-hhc6d, hostIfaceName: afdd8dada842306, network: default, NAD default, SandboxID: "afdd8dada842306ee995d2d5d3572a1e0dd8d5deff2195370db6b98f6bed9fee", PCI device ID: , UID: "d26c6ff8-22dd-4871-bcbe-0568ad1fbcf9", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0325 10:53:53.561924 3561 cni.go:424] [openshift-network-diagnostics/network-check-target-hhc6d afdd8dada842306ee995d2d5d3572a1e0dd8d5deff2195370db6b98f6bed9fee network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-hhc6d afdd8dada842306ee995d2d5d3572a1e0dd8d5deff2195370db6b98f6bed9fee network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:1c:da:f0:05:f8\",\"name\":\"afdd8dada842306\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9601d82-56c0-40b5-8527-4b8353639fe9\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:54.002992 3561 cni.go:403] [kube-system/global-pull-secret-syncer-qlhc8 07a9457100a276ce7feeb11fb9ab5ccd9de52cf5360f45e39e0bb8d5b40430d8 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-qlhc8 07a9457100a276ce7feeb11fb9ab5ccd9de52cf5360f45e39e0bb8d5b40430d8 network default NAD default] I0325 10:53:54.013898 3561 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-qlhc8, hostIfaceName: 07a9457100a276c, network: default, NAD default, SandboxID: "07a9457100a276ce7feeb11fb9ab5ccd9de52cf5360f45e39e0bb8d5b40430d8", PCI device ID: , UID: "8594fb73-a7ca-4f26-802f-51b99b26ebbb", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0325 10:53:54.064669 3561 cni.go:424] [kube-system/global-pull-secret-syncer-qlhc8 07a9457100a276ce7feeb11fb9ab5ccd9de52cf5360f45e39e0bb8d5b40430d8 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-qlhc8 07a9457100a276ce7feeb11fb9ab5ccd9de52cf5360f45e39e0bb8d5b40430d8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:ef:9b:fd:e3:09\",\"name\":\"07a9457100a276c\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ecb5a336-baaa-4f66-be8e-9be6af915de5\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:55.252015 3561 cni.go:403] [openshift-insights/insights-runtime-extractor-wmtzd 4517fa0bee34c6f44ea6a082dbfaf44e24e0963d6bd8b5a072fe87678be6ebf0 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-wmtzd 4517fa0bee34c6f44ea6a082dbfaf44e24e0963d6bd8b5a072fe87678be6ebf0 network default NAD default] I0325 10:53:55.260376 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-wmtzd, hostIfaceName: 4517fa0bee34c6f, network: default, NAD default, SandboxID: "4517fa0bee34c6f44ea6a082dbfaf44e24e0963d6bd8b5a072fe87678be6ebf0", PCI device ID: , UID: "8ee59e44-64cf-4d82-836d-700ec978dc4d", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0325 10:53:55.308789 3561 cni.go:424] [openshift-insights/insights-runtime-extractor-wmtzd 4517fa0bee34c6f44ea6a082dbfaf44e24e0963d6bd8b5a072fe87678be6ebf0 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-wmtzd 4517fa0bee34c6f44ea6a082dbfaf44e24e0963d6bd8b5a072fe87678be6ebf0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:6d:97:de:f8:e7\",\"name\":\"4517fa0bee34c6f\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8e21dec-d6d6-4591-b180-8c793d928de1\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:56.953634 3561 cni.go:403] [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default] I0325 10:53:56.964615 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-5d466cd54f-g75n8, hostIfaceName: c888d76231b7ffb, network: default, NAD default, SandboxID: "c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764", PCI device ID: , UID: "e1e7d6b2-720d-49d8-bb32-71736f6a8c2b", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0325 10:53:57.028816 3561 cni.go:424] [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:2d:2e:84:9c:e6\",\"name\":\"c888d76231b7ffb\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2534d432-9b66-4bab-a4ab-586b762da9a5\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.049105 3561 cni.go:403] [openshift-dns/dns-default-44hsd c2e43c7b2e3f17c995ba115f2f3198eac9c2453c93abc4b1476398d541076e4a network default NAD default] ADD starting CNI request [openshift-dns/dns-default-44hsd c2e43c7b2e3f17c995ba115f2f3198eac9c2453c93abc4b1476398d541076e4a network default NAD default] I0325 10:53:57.057442 3561 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd bd803cb7ce187097520c4fc03495cbbee96444ffec9097e92c1b9cdd935535cc network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd bd803cb7ce187097520c4fc03495cbbee96444ffec9097e92c1b9cdd935535cc network default NAD default] I0325 10:53:57.061288 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-44hsd, hostIfaceName: c2e43c7b2e3f17c, network: default, NAD default, SandboxID: "c2e43c7b2e3f17c995ba115f2f3198eac9c2453c93abc4b1476398d541076e4a", PCI device ID: , UID: "729b5a4f-f447-4463-8ff9-f2e309022923", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0325 10:53:57.068507 3561 cni.go:403] [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default] ADD starting CNI request [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default] I0325 10:53:57.077480 3561 cni.go:403] [openshift-ingress/router-default-bd8d6ccd8-tmbbk 5c2723835a22a8cd3a9960b238c7aec80c52734193cb9d990e0e872c9999abf7 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-bd8d6ccd8-tmbbk 5c2723835a22a8cd3a9960b238c7aec80c52734193cb9d990e0e872c9999abf7 network default NAD default] I0325 10:53:57.088306 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6b966944cc-nrfcl, hostIfaceName: cc92e3c8fdb1b1a, network: default, NAD default, SandboxID: "cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb", PCI device ID: , UID: "de85f53c-6deb-490e-9236-214ddad70b0b", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0325 10:53:57.088499 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-lcxfd, hostIfaceName: bd803cb7ce18709, network: default, NAD default, SandboxID: "bd803cb7ce187097520c4fc03495cbbee96444ffec9097e92c1b9cdd935535cc", PCI device ID: , UID: "1868c840-44f7-4427-bee5-4f6718351c88", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0325 10:53:57.102038 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-bd8d6ccd8-tmbbk, hostIfaceName: 5c2723835a22a8c, network: default, NAD default, SandboxID: "5c2723835a22a8cd3a9960b238c7aec80c52734193cb9d990e0e872c9999abf7", PCI device ID: , UID: "14b2243b-8e63-4b96-819d-2f40c5aa3563", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0325 10:53:57.125135 3561 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn 75020ef0f8db6a6f1bbd4fdbd30cf5194b1f836e6e683de8f0acac3d493eefa9 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn 75020ef0f8db6a6f1bbd4fdbd30cf5194b1f836e6e683de8f0acac3d493eefa9 network default NAD default] I0325 10:53:57.125259 3561 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx 433e14f184c44f1b54a0ffebef247f1c8f623f7998d7d959f940d07170b4b536 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx 433e14f184c44f1b54a0ffebef247f1c8f623f7998d7d959f940d07170b4b536 network default NAD default] I0325 10:53:57.149699 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-bm6cx, hostIfaceName: 433e14f184c44f1, network: default, NAD default, SandboxID: "433e14f184c44f1b54a0ffebef247f1c8f623f7998d7d959f940d07170b4b536", PCI device ID: , UID: "f6032f55-4ad1-4a96-8a93-acc41f9c0b90", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0325 10:53:57.161797 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-b8jfn, hostIfaceName: 75020ef0f8db6a6, network: default, NAD default, SandboxID: "75020ef0f8db6a6f1bbd4fdbd30cf5194b1f836e6e683de8f0acac3d493eefa9", PCI device ID: , UID: "c97bba70-662f-41e1-8da5-5285d1d5483a", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0325 10:53:57.162351 3561 cni.go:424] [openshift-dns/dns-default-44hsd c2e43c7b2e3f17c995ba115f2f3198eac9c2453c93abc4b1476398d541076e4a network default NAD default] ADD finished CNI request [openshift-dns/dns-default-44hsd c2e43c7b2e3f17c995ba115f2f3198eac9c2453c93abc4b1476398d541076e4a network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:e6:9f:49:5b:d6\",\"name\":\"c2e43c7b2e3f17c\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38cbaa20-9702-4e64-9a7a-f9224598c63e\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.190758 3561 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7 8c13b3010b0fb13b9233c9dcec5af20545a8963b506faad41f65e79b2a0d842d network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7 8c13b3010b0fb13b9233c9dcec5af20545a8963b506faad41f65e79b2a0d842d network default NAD default] I0325 10:53:57.213897 3561 cni.go:424] [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default] ADD finished CNI request [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:f9:d3:0f:0a:00\",\"name\":\"cc92e3c8fdb1b1a\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e2531f7-ab18-432f-8323-9b307dbf5648\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.215375 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-9v6q7, hostIfaceName: 8c13b3010b0fb13, network: default, NAD default, SandboxID: "8c13b3010b0fb13b9233c9dcec5af20545a8963b506faad41f65e79b2a0d842d", PCI device ID: , UID: "0203bdd7-a353-4d6b-899e-5c722d8b9ef2", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0325 10:53:57.234092 3561 cni.go:424] [openshift-ingress/router-default-bd8d6ccd8-tmbbk 5c2723835a22a8cd3a9960b238c7aec80c52734193cb9d990e0e872c9999abf7 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-bd8d6ccd8-tmbbk 5c2723835a22a8cd3a9960b238c7aec80c52734193cb9d990e0e872c9999abf7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:9e:48:a9:70:e6\",\"name\":\"5c2723835a22a8c\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6244c965-6315-4312-9ad8-5e839d739950\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.293737 3561 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7 8c13b3010b0fb13b9233c9dcec5af20545a8963b506faad41f65e79b2a0d842d network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-9v6q7 8c13b3010b0fb13b9233c9dcec5af20545a8963b506faad41f65e79b2a0d842d network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:be:fc:f9:4c:a7\",\"name\":\"8c13b3010b0fb13\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de9d2ec0-0d28-4650-9b3c-e9f4168b8bc8\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.432780 3561 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd bd803cb7ce187097520c4fc03495cbbee96444ffec9097e92c1b9cdd935535cc network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-lcxfd bd803cb7ce187097520c4fc03495cbbee96444ffec9097e92c1b9cdd935535cc network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:b0:e3:01:7b:70\",\"name\":\"bd803cb7ce18709\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f73b0bdb-5e03-4e81-b823-4cb4b6db6da7\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.466867 3561 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx 433e14f184c44f1b54a0ffebef247f1c8f623f7998d7d959f940d07170b4b536 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-bm6cx 433e14f184c44f1b54a0ffebef247f1c8f623f7998d7d959f940d07170b4b536 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:2a:56:ad:74:48\",\"name\":\"433e14f184c44f1\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b22c3d47-e2f6-496e-ac4c-6688e99f679e\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:53:57.467048 3561 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn 75020ef0f8db6a6f1bbd4fdbd30cf5194b1f836e6e683de8f0acac3d493eefa9 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-b8jfn 75020ef0f8db6a6f1bbd4fdbd30cf5194b1f836e6e683de8f0acac3d493eefa9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:e0:0f:d5:22:60\",\"name\":\"75020ef0f8db6a6\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/98251499-226f-41ea-8c0e-0d0b604e5beb\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 10:53:59.555482 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0325 10:53:59.556253 3561 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:53:59.559344 3561 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0325 10:53:59.560110 3561 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0325 10:53:59.560173 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0325 10:53:59.560204 3561 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0325 10:53:59.636662 3561 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:53:59.971916 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0325 10:53:59.972259 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0325 10:53:59.975732 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:53:59.975803 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:53:59.975835 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.024732 3561 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:54:00.159745 3561 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:54:00.169698 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0325 10:54:00.169719 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0325 10:54:00.170733 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.170754 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.170763 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.200723 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0325 10:54:00.200795 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0325 10:54:00.202800 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.202822 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.202832 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:00.236278 3561 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0325 10:54:01.029498 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:01.029586 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:01.029600 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0325 10:54:01.029783 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0325 10:54:01.029841 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default I0325 10:54:01.132838 3561 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0325 10:54:01.219829 3561 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0325 10:54:01.367066 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0325 10:54:05.169309 3561 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0325 10:54:07.175001 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0325 10:54:07.182842 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0325 10:54:09.662918 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:54:09.662969 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:54:12.236046 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:12.236086 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:12.236094 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:12.236117 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:12.236125 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:12.236148 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:14.030929 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:14.030966 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:14.030972 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0325 10:54:19.240894 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0325 10:54:19.249065 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0325 10:54:19.479984 3561 base_network_controller_pods.go:487] [default/openshift-console/console-5fd995c466-qw9ks] creating logical port openshift-console_console-5fd995c466-qw9ks for pod on switch ip-10-0-0-91.ec2.internal I0325 10:54:19.480095 3561 kube.go:256] Updating pod openshift-console/console-5fd995c466-qw9ks I0325 10:54:19.497672 3561 pod.go:62] [openshift-console/console-5fd995c466-qw9ks] pod update took 17.604113ms I0325 10:54:19.498339 3561 base_network_controller_pods.go:937] [default/openshift-console/console-5fd995c466-qw9ks] addLogicalPort annotation time took 18.275918ms I0325 10:54:19.499700 3561 pods.go:271] [openshift-console/console-5fd995c466-qw9ks] addLogicalPort took 19.770542ms, libovsdb time 1.027209ms I0325 10:54:19.842170 3561 cni.go:403] [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default] ADD starting CNI request [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default] I0325 10:54:19.850997 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5fd995c466-qw9ks, hostIfaceName: ec6b019dc9f3140, network: default, NAD default, SandboxID: "ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2", PCI device ID: , UID: "c6824436-9be6-4e8e-8d78-5f287fffab00", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0325 10:54:19.895613 3561 cni.go:424] [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default] ADD finished CNI request [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:36:b6:e3:f6:5e\",\"name\":\"ec6b019dc9f3140\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/162cfd5b-f546-4fa5-b268-79d2c0024c61\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 10:54:21.160458 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:54:21.160561 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0325 10:54:22.021013 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0325 10:54:23.078220 3561 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl for pod on switch ip-10-0-0-91.ec2.internal I0325 10:54:23.078349 3561 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl I0325 10:54:23.092319 3561 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl] pod update took 14.008301ms I0325 10:54:23.092579 3561 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl] addLogicalPort annotation time took 14.274995ms I0325 10:54:23.094137 3561 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl] addLogicalPort took 15.938559ms, libovsdb time 1.053781ms W0325 10:54:23.106958 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0325 10:54:23.441689 3561 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default] I0325 10:54:23.450391 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl, hostIfaceName: 1b1ce10ebb17018, network: default, NAD default, SandboxID: "1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91", PCI device ID: , UID: "bfda9d1a-f9ef-4de9-8f39-4895415ca135", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0325 10:54:23.497426 3561 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:94:dd:ee:44:f1\",\"name\":\"1b1ce10ebb17018\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4774e727-8c10-4f6f-a634-14fd8c7bcd21\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 10:54:24.933258 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0325 10:54:26.594415 3561 obj_retry.go:464] Detected object openshift-image-registry/image-registry-85b495c994-s2znv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:26.595261 3561 obj_retry.go:464] Detected object openshift-image-registry/image-registry-85b495c994-s2znv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:26.789369 3561 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5d466cd54f-vxvns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:26.790060 3561 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5d466cd54f-vxvns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:27.328690 3561 obj_retry.go:458] Detected object openshift-image-registry/image-registry-85b495c994-s2znv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:27.328737 3561 obj_retry.go:458] Detected object openshift-image-registry/image-registry-85b495c994-s2znv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:27.333093 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-85b495c994-s2znv I0325 10:54:27.333120 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-85b495c994-s2znv W0325 10:54:27.386783 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0325 10:54:27.386801 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0325 10:54:27.400656 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0325 10:54:27.432988 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0325 10:54:27.458529 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0325 10:54:27.474552 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0325 10:54:27.474566 3561 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0325 10:54:27.474573 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0325 10:54:27.536951 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0325 10:54:27.536975 3561 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0325 10:54:27.536985 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0325 10:54:27.565137 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0325 10:54:27.570960 3561 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5d466cd54f-vxvns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:27.571039 3561 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5d466cd54f-vxvns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:27.580348 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5d466cd54f-vxvns I0325 10:54:27.580376 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5d466cd54f-vxvns W0325 10:54:27.615500 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0325 10:54:27.615521 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0325 10:54:27.845204 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0325 10:54:27.854258 3561 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-5fd995c466-5xxm9 IP address from the namespace address-set, err: pod openshift-console/console-5fd995c466-5xxm9: no pod IPs found I0325 10:54:30.195841 3561 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0325 10:54:30.208671 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0325 10:54:31.368811 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.368829 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.368835 3561 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.368840 3561 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.425685 3561 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.425719 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.425725 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0325 10:54:31.425730 3561 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier I0325 10:54:41.193738 3561 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default] I0325 10:54:41.266547 3561 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl 1b1ce10ebb17018b737b14f58f8d57310ee8eb05b65d7c1f7299b2821a71df91 network default NAD default], result "{}", err I0325 10:54:41.312114 3561 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:41.312158 3561 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl I0325 10:54:41.313755 3561 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl, ips: 10.128.0.25 I0325 10:54:41.313791 3561 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:41.313822 3561 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 10:54:41.313835 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0325 10:54:41.314255 3561 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 10:54:41.314342 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0325 10:54:42.202856 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0325 10:54:42.202921 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0325 10:54:42.202975 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0325 10:54:42.217523 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0325 10:54:42.217644 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0325 10:54:42.217647 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0325 10:54:42.341625 3561 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:42.341734 3561 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:42.341757 3561 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:42.341778 3561 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390cq4hl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:43.155312 3561 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0325 10:54:43.166669 3561 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0325 10:54:43.223271 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0325 10:54:43.261425 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0325 10:54:44.784660 3561 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-kph95 for pod on switch ip-10-0-0-91.ec2.internal I0325 10:54:44.784767 3561 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95 I0325 10:54:44.804034 3561 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95] pod update took 19.2965ms I0325 10:54:44.804131 3561 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95] addLogicalPort annotation time took 19.397427ms I0325 10:54:44.805394 3561 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95] addLogicalPort took 20.747851ms, libovsdb time 911.315µs I0325 10:54:45.144961 3561 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95 45bc4653f6c8a26e515c130d5bd1d335c9ce42e237e62fc3b3a3716cf9b2e0b3 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95 45bc4653f6c8a26e515c130d5bd1d335c9ce42e237e62fc3b3a3716cf9b2e0b3 network default NAD default] I0325 10:54:45.153522 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-kph95, hostIfaceName: 45bc4653f6c8a26, network: default, NAD default, SandboxID: "45bc4653f6c8a26e515c130d5bd1d335c9ce42e237e62fc3b3a3716cf9b2e0b3", PCI device ID: , UID: "6767b118-33cc-42f7-9f2a-bf9201d9cd73", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0325 10:54:45.199432 3561 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95 45bc4653f6c8a26e515c130d5bd1d335c9ce42e237e62fc3b3a3716cf9b2e0b3 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-kph95 45bc4653f6c8a26e515c130d5bd1d335c9ce42e237e62fc3b3a3716cf9b2e0b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:53:2d:f7:b1:ea\",\"name\":\"45bc4653f6c8a26\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/048f92a5-9608-4890-a550-6204b0655dc7\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 10:54:48.239432 3561 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0325 10:54:52.587466 3561 obj_retry.go:464] Detected object openshift-console/console-5bdfbbdcd7-sz4hk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:52.588048 3561 obj_retry.go:464] Detected object openshift-console/console-5bdfbbdcd7-sz4hk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:53.486160 3561 obj_retry.go:458] Detected object openshift-console/console-5bdfbbdcd7-sz4hk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:53.486181 3561 obj_retry.go:458] Detected object openshift-console/console-5bdfbbdcd7-sz4hk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:53.491581 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bdfbbdcd7-sz4hk I0325 10:54:53.491607 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bdfbbdcd7-sz4hk I0325 10:54:54.673342 3561 obj_retry.go:464] Detected object openshift-console/console-5bdfbbdcd7-6hg58 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:54.673852 3561 obj_retry.go:464] Detected object openshift-console/console-5bdfbbdcd7-6hg58 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:54:55.550786 3561 obj_retry.go:458] Detected object openshift-console/console-5bdfbbdcd7-6hg58 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:55.550865 3561 obj_retry.go:458] Detected object openshift-console/console-5bdfbbdcd7-6hg58 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:54:55.558876 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5bdfbbdcd7-6hg58 I0325 10:54:55.558963 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5bdfbbdcd7-6hg58 W0325 10:54:58.286002 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0325 10:54:58.286085 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0325 10:54:58.286009 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0325 10:54:59.247892 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0325 10:54:59.306109 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0325 10:55:15.558843 3561 cni.go:403] [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default] I0325 10:55:15.632902 3561 cni.go:424] [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-5d466cd54f-g75n8 c888d76231b7ffb50f648a63e1da49ed4e9cbb270fb7a2c2a090a2fd39f83764 network default NAD default], result "{}", err I0325 10:55:15.681377 3561 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5d466cd54f-g75n8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:15.681411 3561 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5d466cd54f-g75n8 I0325 10:55:15.682766 3561 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5d466cd54f-g75n8, ips: 10.128.0.6 I0325 10:55:15.682801 3561 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5d466cd54f-g75n8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:16.225025 3561 cni.go:403] [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default] DEL starting CNI request [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default] I0325 10:55:16.298253 3561 cni.go:424] [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default] DEL finished CNI request [openshift-ingress/router-default-6b966944cc-nrfcl cc92e3c8fdb1b1afff72aea90a2ebb395a54203cc6e50a38efa70ce8654595bb network default NAD default], result "{}", err I0325 10:55:16.336112 3561 obj_retry.go:464] Detected object openshift-ingress/router-default-6b966944cc-nrfcl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:16.336167 3561 pods.go:176] Deleting pod: openshift-ingress/router-default-6b966944cc-nrfcl I0325 10:55:16.337590 3561 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-6b966944cc-nrfcl, ips: 10.128.0.12 I0325 10:55:16.337619 3561 obj_retry.go:464] Detected object openshift-ingress/router-default-6b966944cc-nrfcl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:16.379230 3561 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5d466cd54f-g75n8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:16.379311 3561 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5d466cd54f-g75n8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:16.384357 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5d466cd54f-g75n8 I0325 10:55:16.384446 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5d466cd54f-g75n8 I0325 10:55:16.678454 3561 obj_retry.go:458] Detected object openshift-ingress/router-default-6b966944cc-nrfcl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:16.678478 3561 obj_retry.go:458] Detected object openshift-ingress/router-default-6b966944cc-nrfcl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:16.683440 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-6b966944cc-nrfcl I0325 10:55:16.683467 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-6b966944cc-nrfcl I0325 10:55:28.734587 3561 cni.go:403] [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default] DEL starting CNI request [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default] I0325 10:55:28.795424 3561 cni.go:424] [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default] DEL finished CNI request [openshift-console/console-5fd995c466-qw9ks ec6b019dc9f314000d5fe64a7eb9c0f59f063a759bcaba0f0c7455c7b97c53e2 network default NAD default], result "{}", err I0325 10:55:28.843758 3561 obj_retry.go:464] Detected object openshift-console/console-5fd995c466-qw9ks of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:28.843794 3561 pods.go:176] Deleting pod: openshift-console/console-5fd995c466-qw9ks I0325 10:55:28.845390 3561 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5fd995c466-qw9ks, ips: 10.128.0.24 I0325 10:55:28.845419 3561 obj_retry.go:464] Detected object openshift-console/console-5fd995c466-qw9ks of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:29.440291 3561 obj_retry.go:458] Detected object openshift-console/console-5fd995c466-qw9ks of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:29.440317 3561 obj_retry.go:458] Detected object openshift-console/console-5fd995c466-qw9ks of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:29.444817 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5fd995c466-qw9ks I0325 10:55:29.444847 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5fd995c466-qw9ks I0325 10:55:32.584253 3561 base_network_controller_pods.go:487] [default/openshift-console/console-85966d6dc4-jwqjg] creating logical port openshift-console_console-85966d6dc4-jwqjg for pod on switch ip-10-0-0-91.ec2.internal I0325 10:55:32.584389 3561 kube.go:256] Updating pod openshift-console/console-85966d6dc4-jwqjg I0325 10:55:32.609101 3561 pod.go:62] [openshift-console/console-85966d6dc4-jwqjg] pod update took 24.739061ms I0325 10:55:32.609199 3561 base_network_controller_pods.go:937] [default/openshift-console/console-85966d6dc4-jwqjg] addLogicalPort annotation time took 24.837627ms I0325 10:55:32.610887 3561 pods.go:271] [openshift-console/console-85966d6dc4-jwqjg] addLogicalPort took 26.649023ms, libovsdb time 1.167728ms I0325 10:55:32.946366 3561 cni.go:403] [openshift-console/console-85966d6dc4-jwqjg 061ca1eac237843b7b039508db27379a035199ae48138f9e8b8170b45c8487bf network default NAD default] ADD starting CNI request [openshift-console/console-85966d6dc4-jwqjg 061ca1eac237843b7b039508db27379a035199ae48138f9e8b8170b45c8487bf network default NAD default] I0325 10:55:32.954915 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-85966d6dc4-jwqjg, hostIfaceName: 061ca1eac237843, network: default, NAD default, SandboxID: "061ca1eac237843b7b039508db27379a035199ae48138f9e8b8170b45c8487bf", PCI device ID: , UID: "d2a1952f-177a-4897-ad01-c32733df8c77", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0325 10:55:33.001117 3561 cni.go:424] [openshift-console/console-85966d6dc4-jwqjg 061ca1eac237843b7b039508db27379a035199ae48138f9e8b8170b45c8487bf network default NAD default] ADD finished CNI request [openshift-console/console-85966d6dc4-jwqjg 061ca1eac237843b7b039508db27379a035199ae48138f9e8b8170b45c8487bf network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:64:ec:02:07:e4\",\"name\":\"061ca1eac237843\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3f68382-c550-4ac8-9ac3-848fee9d4b91\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 10:55:33.848998 3561 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:33.849537 3561 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:34.717962 3561 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:34.717985 3561 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:34.724239 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0325 10:55:34.724263 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 W0325 10:55:43.514291 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0325 10:55:45.550301 3561 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:45.550903 3561 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:45.798130 3561 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:45.798152 3561 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:45.805455 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0325 10:55:45.805483 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0325 10:55:56.479678 3561 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:56.480234 3561 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:56.744107 3561 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:56.744152 3561 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:56.749606 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0325 10:55:56.749633 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0325 10:55:57.882401 3561 obj_retry.go:464] Detected object openshift-console/console-586dd9bc44-q7tl2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:57.883017 3561 obj_retry.go:464] Detected object openshift-console/console-586dd9bc44-q7tl2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:55:58.770298 3561 obj_retry.go:458] Detected object openshift-console/console-586dd9bc44-q7tl2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:58.770319 3561 obj_retry.go:458] Detected object openshift-console/console-586dd9bc44-q7tl2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:55:58.775448 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-586dd9bc44-q7tl2 I0325 10:55:58.775512 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-586dd9bc44-q7tl2 I0325 10:56:06.916961 3561 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0325 10:56:08.837271 3561 obj_retry.go:464] Detected object openshift-console/console-586dd9bc44-72czr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:56:08.837783 3561 obj_retry.go:464] Detected object openshift-console/console-586dd9bc44-72czr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:56:08.906571 3561 obj_retry.go:458] Detected object openshift-console/console-586dd9bc44-72czr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:56:08.906597 3561 obj_retry.go:458] Detected object openshift-console/console-586dd9bc44-72czr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:56:08.911852 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-586dd9bc44-72czr I0325 10:56:08.911876 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-586dd9bc44-72czr I0325 10:56:18.657830 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default] I0325 10:56:18.727092 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s 96374716645acfe6e4bbb7568b26c579e53dadf3f864caab537a3ab0e3949949 network default NAD default], result "{}", err I0325 10:56:18.770567 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:56:18.770596 3561 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s I0325 10:56:18.771953 3561 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s, ips: 10.128.0.19 I0325 10:56:18.771987 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:56:19.669738 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:56:19.669765 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:56:48.397037 3561 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 10:56:48.397606 3561 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 10:56:48.972485 3561 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:56:48.972508 3561 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 10:56:48.981408 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0325 10:56:48.981424 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0325 10:59:04.112720 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0325 10:59:31.139903 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0325 10:59:36.815174 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0325 10:59:49.824693 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=637 I0325 11:00:00.197176 3561 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29573940-7v5jp for pod on switch ip-10-0-0-91.ec2.internal I0325 11:00:00.197298 3561 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp I0325 11:00:00.223119 3561 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp] pod update took 25.845556ms I0325 11:00:00.223216 3561 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp] addLogicalPort annotation time took 25.947826ms I0325 11:00:00.224497 3561 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp] addLogicalPort took 27.338455ms, libovsdb time 888.496µs I0325 11:00:00.573773 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default] I0325 11:00:00.583310 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29573940-7v5jp, hostIfaceName: 3ba0568779119fa, network: default, NAD default, SandboxID: "3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655", PCI device ID: , UID: "3dfdd0fb-44f0-464b-a1c3-97e4c6d35c63", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0325 11:00:00.628835 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:61:c9:8d:e2:ff\",\"name\":\"3ba0568779119fa\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b12ceb51-3dc0-451c-b2ed-ce4335781638\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:00:03.539345 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default] I0325 11:00:03.600593 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp 3ba0568779119fa8e267e9ae3986f9dc67bc3b234aef7ecea60741059c981655 network default NAD default], result "{}", err I0325 11:00:03.642587 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:00:03.642629 3561 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp I0325 11:00:03.644021 3561 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp, ips: 10.128.0.28 I0325 11:00:03.644055 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:00:04.548081 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:00:04.548135 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573940-7v5jp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:00:18.920569 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0325 11:00:38.814261 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0325 11:01:11.822372 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=15 I0325 11:01:26.922048 3561 namespace.go:98] [tsf] adding namespace I0325 11:01:26.923056 3561 namespace.go:102] [tsf] adding namespace took 913.511µs I0325 11:01:26.940864 3561 namespace.go:142] [tsf] updating namespace I0325 11:01:26.953219 3561 namespace.go:142] [tsf] updating namespace I0325 11:01:30.766802 3561 namespace.go:98] [konflux-operator] adding namespace I0325 11:01:30.767741 3561 namespace.go:102] [konflux-operator] adding namespace took 901.603µs I0325 11:01:30.771665 3561 namespace.go:98] [rhtpa-operator] adding namespace I0325 11:01:30.771810 3561 namespace.go:98] [tsf-tpa] adding namespace I0325 11:01:30.772548 3561 namespace.go:102] [rhtpa-operator] adding namespace took 813.271µs I0325 11:01:30.773206 3561 namespace.go:102] [tsf-tpa] adding namespace took 1.380201ms I0325 11:01:30.774897 3561 namespace.go:98] [cert-manager-operator] adding namespace I0325 11:01:30.775028 3561 namespace.go:98] [tsf-tas] adding namespace I0325 11:01:30.775174 3561 namespace.go:98] [rhbk-operator] adding namespace I0325 11:01:30.775268 3561 namespace.go:98] [tsf-keycloak] adding namespace I0325 11:01:30.775356 3561 namespace.go:98] [konflux-ui] adding namespace I0325 11:01:30.775713 3561 namespace.go:102] [cert-manager-operator] adding namespace took 801.186µs I0325 11:01:30.776219 3561 namespace.go:102] [tsf-tas] adding namespace took 1.153818ms I0325 11:01:30.776961 3561 namespace.go:102] [rhbk-operator] adding namespace took 1.733271ms I0325 11:01:30.777452 3561 namespace.go:102] [tsf-keycloak] adding namespace took 2.155007ms I0325 11:01:30.778069 3561 namespace.go:102] [konflux-ui] adding namespace took 2.670145ms I0325 11:01:30.788257 3561 namespace.go:142] [konflux-operator] updating namespace I0325 11:01:30.806898 3561 namespace.go:142] [konflux-operator] updating namespace I0325 11:01:30.818323 3561 namespace.go:142] [rhtpa-operator] updating namespace I0325 11:01:30.818421 3561 namespace.go:142] [cert-manager-operator] updating namespace I0325 11:01:30.822001 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:01:30.822019 3561 namespace.go:142] [tsf-tas] updating namespace I0325 11:01:30.835995 3561 namespace.go:142] [rhbk-operator] updating namespace I0325 11:01:30.836021 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:01:30.840262 3561 namespace.go:142] [konflux-operator] updating namespace I0325 11:01:30.840305 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:01:30.881850 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:01:30.887616 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:01:30.919868 3561 namespace.go:142] [cert-manager-operator] updating namespace I0325 11:01:30.935046 3561 namespace.go:142] [cert-manager-operator] updating namespace I0325 11:01:30.944811 3561 namespace.go:142] [tsf-tas] updating namespace I0325 11:01:30.956412 3561 namespace.go:142] [tsf-tas] updating namespace I0325 11:01:30.970218 3561 namespace.go:142] [rhbk-operator] updating namespace I0325 11:01:30.986829 3561 namespace.go:142] [rhbk-operator] updating namespace I0325 11:01:31.014253 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:01:31.047927 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:01:31.054632 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:01:31.075413 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:01:31.089751 3561 namespace.go:142] [rhtpa-operator] updating namespace I0325 11:01:31.110955 3561 namespace.go:142] [rhtpa-operator] updating namespace I0325 11:01:32.043266 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:01:32.049205 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:01:32.106999 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0325 11:01:32.212232 3561 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:32.212379 3561 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0325 11:01:32.241416 3561 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 29.073998ms I0325 11:01:32.241440 3561 base_network_controller_pods.go:937] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 29.10233ms I0325 11:01:32.243031 3561 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 30.812261ms, libovsdb time 1.226638ms I0325 11:01:32.584825 3561 cni.go:403] [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default] ADD starting CNI request [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default] I0325 11:01:32.594928 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 7ded994a23d2e90, network: default, NAD default, SandboxID: "7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0", PCI device ID: , UID: "7bc2a0c2-b977-4a28-8431-04804927baa3", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0325 11:01:32.638319 3561 cni.go:424] [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default] ADD finished CNI request [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:df:0c:77:2e:a7\",\"name\":\"7ded994a23d2e90\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9a1be71d-2fa4-4115-a029-0ec2976209ed\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:33.857856 3561 base_network_controller_pods.go:487] [default/openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn] creating logical port openshift-marketplace_7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:33.858034 3561 kube.go:256] Updating pod openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn I0325 11:01:33.872697 3561 pod.go:62] [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn] pod update took 14.701147ms I0325 11:01:33.872728 3561 base_network_controller_pods.go:937] [default/openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn] addLogicalPort annotation time took 14.737566ms I0325 11:01:33.874616 3561 pods.go:271] [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn] addLogicalPort took 16.778145ms, libovsdb time 1.417921ms I0325 11:01:33.952857 3561 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:33.953002 3561 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t I0325 11:01:33.966684 3561 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t] pod update took 13.707764ms I0325 11:01:33.966776 3561 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t] addLogicalPort annotation time took 13.803127ms I0325 11:01:33.968615 3561 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t] addLogicalPort took 15.769374ms, libovsdb time 1.284033ms I0325 11:01:34.225768 3561 cni.go:403] [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default] ADD starting CNI request [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default] I0325 11:01:34.236062 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn, hostIfaceName: e64c51837e13cc1, network: default, NAD default, SandboxID: "e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5", PCI device ID: , UID: "6504e8dc-3532-456c-a77f-1bdf4ef0d7bb", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0325 11:01:34.286523 3561 cni.go:424] [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default] ADD finished CNI request [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:a2:a6:fd:85:46\",\"name\":\"e64c51837e13cc1\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb07fb24-bec8-4100-8bfa-7f1a4a8e0296\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:34.858451 3561 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default] I0325 11:01:34.868391 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t, hostIfaceName: c6e2c967ef3aa0b, network: default, NAD default, SandboxID: "c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d", PCI device ID: , UID: "e14953e6-8948-4167-9fcf-2676ed3374cc", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0325 11:01:34.921991 3561 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:28:bc:bc:4b:81\",\"name\":\"c6e2c967ef3aa0b\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70a8c93b-4818-4d0f-a86e-8825ef8846c6\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:36.873915 3561 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:36.874239 3561 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb I0325 11:01:36.887848 3561 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb] pod update took 13.673975ms I0325 11:01:36.887953 3561 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb] addLogicalPort annotation time took 13.767249ms I0325 11:01:36.893430 3561 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb] addLogicalPort took 19.535178ms, libovsdb time 4.969253ms I0325 11:01:37.100694 3561 base_network_controller_pods.go:487] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt] creating logical port openshift-marketplace_ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:37.100824 3561 kube.go:256] Updating pod openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt I0325 11:01:37.114128 3561 pod.go:62] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt] pod update took 13.342292ms I0325 11:01:37.114216 3561 base_network_controller_pods.go:937] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt] addLogicalPort annotation time took 13.43783ms I0325 11:01:37.116705 3561 pods.go:271] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt] addLogicalPort took 16.032935ms, libovsdb time 1.897372ms I0325 11:01:37.282453 3561 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default] I0325 11:01:37.293555 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb, hostIfaceName: 56d114b97615690, network: default, NAD default, SandboxID: "56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52", PCI device ID: , UID: "8a44d1a3-7fcc-49e7-9419-0e252b6f5a71", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0325 11:01:37.569187 3561 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:3f:6a:61:77:63\",\"name\":\"56d114b97615690\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d38118d-fd61-443c-9e84-627b8ab31aa3\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:38.994376 3561 cni.go:403] [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default] DEL starting CNI request [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default] I0325 11:01:39.076085 3561 cni.go:424] [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default] DEL finished CNI request [openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn e64c51837e13cc1d00a9ecee674d1f92baaf79883267650ad4e172e75755b9d5 network default NAD default], result "{}", err I0325 11:01:39.812724 3561 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default] ADD starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default] I0325 11:01:39.824416 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt, hostIfaceName: 1f1d7cdd80d840f, network: default, NAD default, SandboxID: "1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb", PCI device ID: , UID: "a23b0509-bb5d-4424-bc90-d23208b78ab1", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0325 11:01:39.886668 3561 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default] ADD finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:22:b1:b2:19:0c\",\"name\":\"1f1d7cdd80d840f\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/826efca8-7ff1-49b7-9274-a911e2cf98dc\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:40.072101 3561 obj_retry.go:464] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:40.072173 3561 pods.go:176] Deleting pod: openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn I0325 11:01:40.074230 3561 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn, ips: 10.128.0.30 I0325 11:01:40.074284 3561 obj_retry.go:464] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:40.074315 3561 obj_retry.go:464] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:40.074359 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0325 11:01:40.074732 3561 obj_retry.go:464] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:40.074751 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0325 11:01:40.137559 3561 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:40.137791 3561 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg I0325 11:01:40.150616 3561 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg] pod update took 12.866049ms I0325 11:01:40.150647 3561 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg] addLogicalPort annotation time took 12.903246ms I0325 11:01:40.152279 3561 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg] addLogicalPort took 14.739952ms, libovsdb time 1.170201ms I0325 11:01:40.237093 3561 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z for pod on switch ip-10-0-0-91.ec2.internal I0325 11:01:40.237252 3561 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z I0325 11:01:40.251713 3561 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z] pod update took 14.503723ms I0325 11:01:40.251786 3561 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z] addLogicalPort annotation time took 14.581099ms I0325 11:01:40.253643 3561 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z] addLogicalPort took 16.56849ms, libovsdb time 1.34928ms I0325 11:01:41.098400 3561 obj_retry.go:458] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:41.098498 3561 obj_retry.go:458] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:41.098576 3561 obj_retry.go:458] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:41.098595 3561 obj_retry.go:458] Detected object openshift-marketplace/7091d389759639f2337a48196edf37808d3becb67634b1d22367344bdb26gqn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:42.734025 3561 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default] I0325 11:01:42.737167 3561 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default] I0325 11:01:42.751346 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z, hostIfaceName: b68cdb2b0e4ea97, network: default, NAD default, SandboxID: "b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82", PCI device ID: , UID: "4735094c-9be7-4c7b-bb04-e52d0a8979a8", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0325 11:01:42.751343 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg, hostIfaceName: ad1d90e423632fe, network: default, NAD default, SandboxID: "ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4", PCI device ID: , UID: "a5a5081f-ba25-422f-a3aa-5ff7d035d158", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0325 11:01:42.822950 3561 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:36:f3:69:3f:2a\",\"name\":\"b68cdb2b0e4ea97\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3163b6e-6985-4544-80b2-e88d9604d442\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:42.848114 3561 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:fc:1b:c9:72:0b\",\"name\":\"ad1d90e423632fe\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6ce30c4f-d6ce-4c89-9c28-9582887cd8d4\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:01:45.042768 3561 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default] I0325 11:01:45.113197 3561 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t c6e2c967ef3aa0bc52572a92c508b212360bc8404921f91ab662c34ea092e75d network default NAD default], result "{}", err I0325 11:01:45.161561 3561 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:45.161603 3561 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t I0325 11:01:45.163487 3561 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t, ips: 10.128.0.31 I0325 11:01:45.163525 3561 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:45.163558 3561 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:45.163581 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0325 11:01:45.163862 3561 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:45.163883 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0325 11:01:45.816293 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=75 I0325 11:01:46.056748 3561 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default] DEL starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default] I0325 11:01:46.058847 3561 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default] I0325 11:01:46.119561 3561 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default] DEL finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt 1f1d7cdd80d840ff6d78ac6f5dcad18726e5c319c6d9e26c074e9b17ddd428bb network default NAD default], result "{}", err I0325 11:01:46.180429 3561 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb 56d114b9761569071cd8c42dfbd913a64cef9dd4f83ab1d912ef8b25bfd3df52 network default NAD default], result "{}", err I0325 11:01:46.198103 3561 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.198176 3561 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:46.198200 3561 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:46.198217 3561 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:46.198219 3561 pods.go:176] Deleting pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt I0325 11:01:46.198227 3561 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5bxp6t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:46.200082 3561 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt, ips: 10.128.0.33 I0325 11:01:46.200114 3561 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.200139 3561 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.200149 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0325 11:01:46.200406 3561 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.200422 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0325 11:01:46.234040 3561 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.234194 3561 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb I0325 11:01:46.235609 3561 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb, ips: 10.128.0.32 I0325 11:01:46.235644 3561 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.235671 3561 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.235682 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0325 11:01:46.235905 3561 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:46.235921 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0325 11:01:47.068716 3561 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default] I0325 11:01:47.136563 3561 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z b68cdb2b0e4ea9717ef0e7d885a101da11fc2490688352ef396dffe070f24a82 network default NAD default], result "{}", err I0325 11:01:47.186735 3561 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:47.186770 3561 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z I0325 11:01:47.188395 3561 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z, ips: 10.128.0.35 I0325 11:01:47.188433 3561 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:47.188458 3561 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:47.188466 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0325 11:01:47.188670 3561 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:47.188682 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0325 11:01:47.224318 3561 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.224401 3561 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.224441 3561 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.224470 3561 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sjsnt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.263761 3561 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.263785 3561 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.263815 3561 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:47.263829 3561 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1nvzcb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:48.070150 3561 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default] I0325 11:01:48.139487 3561 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg ad1d90e423632fe4074f35a97b73738f44889f1a323126d9d21f13d47c9582c4 network default NAD default], result "{}", err I0325 11:01:48.201852 3561 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:48.202060 3561 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg I0325 11:01:48.204410 3561 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg, ips: 10.128.0.34 I0325 11:01:48.204451 3561 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:01:48.204484 3561 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:48.204494 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0325 11:01:48.204751 3561 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0325 11:01:48.204769 3561 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0325 11:01:48.216044 3561 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:48.216121 3561 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:48.216162 3561 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:48.216200 3561 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7w562z of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:49.227334 3561 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:49.227460 3561 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:49.227483 3561 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:49.227496 3561 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafnwppg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:01:57.689868 3561 namespace.go:98] [cert-manager] adding namespace I0325 11:01:57.690741 3561 namespace.go:102] [cert-manager] adding namespace took 842.74µs I0325 11:01:57.745745 3561 namespace.go:142] [cert-manager] updating namespace I0325 11:01:57.764837 3561 namespace.go:142] [cert-manager] updating namespace W0325 11:01:58.363877 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0325 11:02:01.742657 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0325 11:02:01.752016 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0325 11:02:01.762326 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0325 11:02:01.770169 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0325 11:02:06.358197 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0325 11:02:06.358218 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0325 11:02:08.390514 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0325 11:02:09.464784 3561 namespace.go:142] [openshift-operators] updating namespace I0325 11:02:09.474461 3561 namespace.go:142] [openshift-operators] updating namespace I0325 11:02:14.236663 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0325 11:02:18.585860 3561 namespace.go:98] [trusted-artifact-signer] adding namespace I0325 11:02:18.586847 3561 namespace.go:102] [trusted-artifact-signer] adding namespace took 954.47µs I0325 11:02:18.612975 3561 namespace.go:142] [trusted-artifact-signer] updating namespace I0325 11:02:18.628150 3561 namespace.go:142] [trusted-artifact-signer] updating namespace W0325 11:02:19.411752 3561 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0325 11:02:21.680883 3561 namespace.go:142] [openshift-operators] updating namespace I0325 11:02:36.312210 3561 namespace.go:98] [openshift-pipelines] adding namespace I0325 11:02:36.313552 3561 namespace.go:102] [openshift-pipelines] adding namespace took 1.220517ms I0325 11:02:36.330012 3561 namespace.go:142] [openshift-pipelines] updating namespace I0325 11:02:36.341697 3561 cni.go:403] [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default] DEL starting CNI request [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default] I0325 11:02:36.412520 3561 cni.go:424] [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default] DEL finished CNI request [tsf/test-tsf-subscriptions 7ded994a23d2e90266128d43f1cb619001de0818e6715038a592b7a1a99638e0 network default NAD default], result "{}", err I0325 11:02:36.462674 3561 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:02:36.462707 3561 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0325 11:02:36.464049 3561 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.29 I0325 11:02:36.464082 3561 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:02:36.484371 3561 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:02:36.484400 3561 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:02:36.489019 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0325 11:02:36.489046 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0325 11:02:37.019399 3561 namespace.go:142] [cert-manager] updating namespace I0325 11:02:37.222318 3561 namespace.go:142] [cert-manager-operator] updating namespace I0325 11:02:37.630527 3561 namespace.go:142] [konflux-operator] updating namespace I0325 11:02:37.691784 3561 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-0-91.ec2.internal I0325 11:02:37.691896 3561 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0325 11:02:37.705739 3561 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 13.877458ms I0325 11:02:37.705768 3561 base_network_controller_pods.go:937] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 13.912575ms I0325 11:02:37.711386 3561 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 19.617692ms, libovsdb time 5.129767ms I0325 11:02:37.826291 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:02:38.026354 3561 namespace.go:142] [open-cluster-management-2p8mq20h6bsbq55jr1en15t0ro8fskh6] updating namespace I0325 11:02:38.067623 3561 cni.go:403] [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default] ADD starting CNI request [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default] I0325 11:02:38.076324 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 26ff10e7855b07f, network: default, NAD default, SandboxID: "26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a", PCI device ID: , UID: "e92f41b2-8133-46f2-aa5e-d15bd57f71f3", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0325 11:02:38.122345 3561 cni.go:424] [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default] ADD finished CNI request [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:82:85:65:53:9a\",\"name\":\"26ff10e7855b07f\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/28ecbfe7-1e3c-45c4-9d1d-facd07065218\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:02:38.428833 3561 namespace.go:142] [rhbk-operator] updating namespace I0325 11:02:38.629976 3561 namespace.go:142] [rhtpa-operator] updating namespace I0325 11:02:38.831209 3561 namespace.go:142] [trusted-artifact-signer] updating namespace I0325 11:02:39.031202 3561 namespace.go:142] [tsf] updating namespace I0325 11:02:39.231010 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:02:39.431892 3561 namespace.go:142] [tsf-tas] updating namespace I0325 11:02:39.635063 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:02:39.758655 3561 namespace.go:142] [cert-manager] updating namespace I0325 11:02:39.766325 3561 namespace.go:142] [cert-manager-operator] updating namespace I0325 11:02:39.782045 3561 namespace.go:142] [default] updating namespace I0325 11:02:39.790160 3561 namespace.go:142] [konflux-operator] updating namespace I0325 11:02:39.799872 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:02:39.807583 3561 namespace.go:142] [open-cluster-management-2p8mq20h6bsbq55jr1en15t0ro8fskh6] updating namespace I0325 11:02:39.821055 3561 namespace.go:142] [openshift] updating namespace I0325 11:02:39.832863 3561 namespace.go:142] [rhbk-operator] updating namespace I0325 11:02:39.844506 3561 namespace.go:142] [rhtpa-operator] updating namespace I0325 11:02:39.856906 3561 namespace.go:142] [trusted-artifact-signer] updating namespace I0325 11:02:39.863644 3561 namespace.go:142] [tsf] updating namespace I0325 11:02:39.873449 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:02:39.894177 3561 namespace.go:142] [tsf-tas] updating namespace I0325 11:02:39.916551 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:02:40.014360 3561 namespace.go:142] [cert-manager] updating namespace I0325 11:02:40.117025 3561 namespace.go:142] [cert-manager-operator] updating namespace I0325 11:02:40.221125 3561 namespace.go:142] [default] updating namespace I0325 11:02:40.319347 3561 namespace.go:142] [konflux-operator] updating namespace I0325 11:02:40.424006 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:02:40.518216 3561 namespace.go:142] [open-cluster-management-2p8mq20h6bsbq55jr1en15t0ro8fskh6] updating namespace I0325 11:02:40.550502 3561 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-zrsrn for pod on switch ip-10-0-0-91.ec2.internal I0325 11:02:40.550678 3561 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn I0325 11:02:40.554632 3561 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-tnm2d for pod on switch ip-10-0-0-91.ec2.internal I0325 11:02:40.554807 3561 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d I0325 11:02:40.563011 3561 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn] pod update took 12.368908ms I0325 11:02:40.563080 3561 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn] addLogicalPort annotation time took 12.441855ms I0325 11:02:40.564710 3561 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn] addLogicalPort took 14.229523ms, libovsdb time 1.129897ms I0325 11:02:40.569285 3561 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d] pod update took 14.513562ms I0325 11:02:40.569312 3561 base_network_controller_pods.go:937] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d] addLogicalPort annotation time took 14.543786ms I0325 11:02:40.571686 3561 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d] addLogicalPort took 17.067934ms, libovsdb time 1.959158ms I0325 11:02:40.613605 3561 namespace.go:142] [openshift] updating namespace I0325 11:02:40.715274 3561 namespace.go:142] [rhbk-operator] updating namespace I0325 11:02:40.819040 3561 namespace.go:142] [rhtpa-operator] updating namespace I0325 11:02:40.916111 3561 namespace.go:142] [trusted-artifact-signer] updating namespace I0325 11:02:41.015954 3561 namespace.go:142] [tsf] updating namespace I0325 11:02:41.116055 3561 namespace.go:142] [tsf-keycloak] updating namespace I0325 11:02:41.216727 3561 namespace.go:142] [tsf-tas] updating namespace I0325 11:02:41.278733 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0325 11:02:41.316977 3561 namespace.go:142] [tsf-tpa] updating namespace I0325 11:02:41.693878 3561 namespace.go:142] [openshift-pipelines] updating namespace I0325 11:02:44.016799 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0325 11:02:45.737618 3561 cni.go:403] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn 661f5eddb12286883a2a2f3c432aa4b68453a3d09f4bc78623f7d08d3a7c150a network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn 661f5eddb12286883a2a2f3c432aa4b68453a3d09f4bc78623f7d08d3a7c150a network default NAD default] I0325 11:02:45.747002 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-zrsrn, hostIfaceName: 661f5eddb122868, network: default, NAD default, SandboxID: "661f5eddb12286883a2a2f3c432aa4b68453a3d09f4bc78623f7d08d3a7c150a", PCI device ID: , UID: "0d0f859e-3be6-4cb4-8306-8d5ee2245bab", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0325 11:02:45.749172 3561 cni.go:403] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d ff4213532fb166051a12338c4c13129d93b2449ecf70c6e58fc367d6cf90756e network default NAD default] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d ff4213532fb166051a12338c4c13129d93b2449ecf70c6e58fc367d6cf90756e network default NAD default] I0325 11:02:45.760288 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-tnm2d, hostIfaceName: ff4213532fb1660, network: default, NAD default, SandboxID: "ff4213532fb166051a12338c4c13129d93b2449ecf70c6e58fc367d6cf90756e", PCI device ID: , UID: "7ecef7aa-3a32-44ed-8bc8-8cb77a12da81", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0325 11:02:45.799551 3561 cni.go:424] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn 661f5eddb12286883a2a2f3c432aa4b68453a3d09f4bc78623f7d08d3a7c150a network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-zrsrn 661f5eddb12286883a2a2f3c432aa4b68453a3d09f4bc78623f7d08d3a7c150a network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:e2:4b:da:63:3f\",\"name\":\"661f5eddb122868\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2262b3de-606e-4f75-a6ae-2a0b35f65817\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:02:45.824821 3561 cni.go:424] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d ff4213532fb166051a12338c4c13129d93b2449ecf70c6e58fc367d6cf90756e network default NAD default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-tnm2d ff4213532fb166051a12338c4c13129d93b2449ecf70c6e58fc367d6cf90756e network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:3f:5b:7c:30:90\",\"name\":\"ff4213532fb1660\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5aa9b7ad-43a9-4f8c-993d-023c98bea45e\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:02:46.399461 3561 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0325 11:02:46.416471 3561 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0325 11:02:50.420220 3561 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:02:50.420467 3561 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6 I0325 11:02:50.440038 3561 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6] pod update took 19.612205ms I0325 11:02:50.440126 3561 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6] addLogicalPort annotation time took 19.704454ms I0325 11:02:50.441425 3561 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6] addLogicalPort took 21.242232ms, libovsdb time 883.115µs I0325 11:02:50.519984 3561 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-jmz9d for pod on switch ip-10-0-0-91.ec2.internal I0325 11:02:50.520139 3561 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d I0325 11:02:50.538694 3561 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d] pod update took 18.595516ms I0325 11:02:50.538727 3561 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d] addLogicalPort annotation time took 18.633251ms I0325 11:02:50.540036 3561 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d] addLogicalPort took 20.07338ms, libovsdb time 897.533µs I0325 11:02:50.622986 3561 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-z7x7p for pod on switch ip-10-0-0-91.ec2.internal I0325 11:02:50.623102 3561 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p I0325 11:02:50.638192 3561 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p] pod update took 15.122446ms I0325 11:02:50.638218 3561 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p] addLogicalPort annotation time took 15.152119ms I0325 11:02:50.639735 3561 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p] addLogicalPort took 16.769294ms, libovsdb time 1.091658ms I0325 11:02:50.786494 3561 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6 78d99babc373a887b9748fd5adc87857b597968eeaa8109940c7582e460f3a80 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6 78d99babc373a887b9748fd5adc87857b597968eeaa8109940c7582e460f3a80 network default NAD default] I0325 11:02:50.796041 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6, hostIfaceName: 78d99babc373a88, network: default, NAD default, SandboxID: "78d99babc373a887b9748fd5adc87857b597968eeaa8109940c7582e460f3a80", PCI device ID: , UID: "03063f6a-a331-42d6-8065-571409b38bf8", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0325 11:02:50.842778 3561 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6 78d99babc373a887b9748fd5adc87857b597968eeaa8109940c7582e460f3a80 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-jt5f6 78d99babc373a887b9748fd5adc87857b597968eeaa8109940c7582e460f3a80 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:60:e2:5c:de:d7\",\"name\":\"78d99babc373a88\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5fa15836-2c70-41b2-87d5-1e999d68a669\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:02:50.886012 3561 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d 16ce6c9b02d346d78ad7896cbd371944367f83eb9fe680cf4f44489be51a671e network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d 16ce6c9b02d346d78ad7896cbd371944367f83eb9fe680cf4f44489be51a671e network default NAD default] I0325 11:02:50.895638 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-jmz9d, hostIfaceName: 16ce6c9b02d346d, network: default, NAD default, SandboxID: "16ce6c9b02d346d78ad7896cbd371944367f83eb9fe680cf4f44489be51a671e", PCI device ID: , UID: "1683311c-ae40-4c2a-85a0-a266ecf61493", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0325 11:02:50.942285 3561 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d 16ce6c9b02d346d78ad7896cbd371944367f83eb9fe680cf4f44489be51a671e network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-jmz9d 16ce6c9b02d346d78ad7896cbd371944367f83eb9fe680cf4f44489be51a671e network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:5f:1a:e0:77:2b\",\"name\":\"16ce6c9b02d346d\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f698316-3266-455a-ad72-2b3b7d8eb383\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:02:50.992698 3561 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p e9ab7bff6b06d86c4a905caabd88f0d780b88e779f9a39800ef149b06802266b network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p e9ab7bff6b06d86c4a905caabd88f0d780b88e779f9a39800ef149b06802266b network default NAD default] I0325 11:02:51.004082 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-z7x7p, hostIfaceName: e9ab7bff6b06d86, network: default, NAD default, SandboxID: "e9ab7bff6b06d86c4a905caabd88f0d780b88e779f9a39800ef149b06802266b", PCI device ID: , UID: "de3b8968-3d31-4cb5-978f-db82f0bc279d", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0325 11:02:51.049521 3561 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p e9ab7bff6b06d86c4a905caabd88f0d780b88e779f9a39800ef149b06802266b network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-z7x7p e9ab7bff6b06d86c4a905caabd88f0d780b88e779f9a39800ef149b06802266b network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:0f:f8:8a:5b:f4\",\"name\":\"e9ab7bff6b06d86\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/82827d0e-22e8-486d-9cc7-5655ba7b819d\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:02:53.606316 3561 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0325 11:02:53.606350 3561 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0325 11:02:53.606357 3561 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0325 11:02:56.566063 3561 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0325 11:02:56.566082 3561 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0325 11:02:56.566087 3561 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0325 11:02:57.512060 3561 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0325 11:02:57.512085 3561 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0325 11:02:57.512091 3561 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0325 11:02:57.512094 3561 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0325 11:03:02.230008 3561 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0325 11:03:02.936323 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0325 11:03:07.411614 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0325 11:03:07.430764 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0325 11:03:08.494487 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0325 11:03:08.494508 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0325 11:03:08.494513 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0325 11:03:08.494518 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0325 11:03:10.556013 3561 cni.go:403] [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default] DEL starting CNI request [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default] I0325 11:03:10.637192 3561 cni.go:424] [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default] DEL finished CNI request [tsf/test-tsf-infrastructure 26ff10e7855b07fea58a4ef80f1f14d086449fcf707e3d72b04921ca8f9ce90a network default NAD default], result "{}", err I0325 11:03:10.694435 3561 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:03:10.694560 3561 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0325 11:03:10.695921 3561 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.36 I0325 11:03:10.695972 3561 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:03:10.720420 3561 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:03:10.720478 3561 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:03:10.726649 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0325 11:03:10.726696 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0325 11:03:11.117535 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0325 11:03:11.804322 3561 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-0-91.ec2.internal I0325 11:03:11.804584 3561 kube.go:256] Updating pod tsf/test-tsf-iam I0325 11:03:11.819242 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=194 I0325 11:03:11.825676 3561 pod.go:62] [tsf/test-tsf-iam] pod update took 21.130892ms I0325 11:03:11.825704 3561 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 21.16683ms I0325 11:03:11.827405 3561 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 23.103719ms, libovsdb time 1.11325ms I0325 11:03:11.906799 3561 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:03:11.906973 3561 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0325 11:03:11.921249 3561 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 14.319841ms I0325 11:03:11.921275 3561 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 14.351502ms I0325 11:03:11.922761 3561 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 15.982025ms, libovsdb time 1.111103ms I0325 11:03:12.180764 3561 cni.go:403] [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default] ADD starting CNI request [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default] I0325 11:03:12.191456 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: f069a8ec72ef545, network: default, NAD default, SandboxID: "f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da", PCI device ID: , UID: "9fdda600-91c1-48a7-8aee-aec93fb1d8ba", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0325 11:03:12.240268 3561 cni.go:424] [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default] ADD finished CNI request [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:83:b2:b4:be:3f\",\"name\":\"f069a8ec72ef545\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b816eed-ac72-4aeb-be8e-c054e3d9843a\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:03:12.270753 3561 cni.go:403] [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default] I0325 11:03:12.281545 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: a1fa35945be727b, network: default, NAD default, SandboxID: "a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69", PCI device ID: , UID: "5ab41f25-1db0-4679-9e07-33ab6bc05127", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0325 11:03:12.328475 3561 cni.go:424] [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:83:47:bc:06:cd\",\"name\":\"a1fa35945be727b\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c53dcba0-a146-43f5-b693-080f7271004a\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:03:19.617634 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0325 11:03:19.617722 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:03:19.617745 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:03:20.693276 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0325 11:03:39.707705 3561 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0325 11:03:40.558788 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:03:40.558882 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:03:40.559058 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0325 11:03:41.887165 3561 cni.go:403] [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default] I0325 11:03:41.953247 3561 cni.go:424] [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 a1fa35945be727bd79a62b85b54c8fb21d1f398fa0bf812d171efc29c6862d69 network default NAD default], result "{}", err I0325 11:03:42.003868 3561 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:03:42.003916 3561 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0325 11:03:42.005250 3561 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.43 I0325 11:03:42.005289 3561 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:03:42.758456 3561 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:03:42.758524 3561 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:03:42.763259 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0325 11:03:42.763329 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0325 11:03:42.799512 3561 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:03:42.799596 3561 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0325 11:03:42.816559 3561 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 16.985888ms I0325 11:03:42.816588 3561 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 17.018262ms I0325 11:03:42.818025 3561 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 18.526238ms, libovsdb time 944.525µs I0325 11:03:43.161110 3561 cni.go:403] [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default] I0325 11:03:43.170817 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 01198b9cd74857e, network: default, NAD default, SandboxID: "01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5", PCI device ID: , UID: "fca137c6-e344-4d06-a59d-df055113095d", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0325 11:03:43.219829 3561 cni.go:424] [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:e1:84:b3:1a:b3\",\"name\":\"01198b9cd74857e\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49872b72-7241-4c2e-b8b2-0a1534122a28\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:03:45.787282 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0325 11:03:45.792119 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:03:45.792230 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:04:01.517549 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:04:01.517584 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:04:01.522909 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0325 11:04:02.201417 3561 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-24gsw] creating logical port tsf-keycloak_tsf-iam-24gsw for pod on switch ip-10-0-0-91.ec2.internal I0325 11:04:02.201566 3561 kube.go:256] Updating pod tsf-keycloak/tsf-iam-24gsw I0325 11:04:02.215766 3561 pod.go:62] [tsf-keycloak/tsf-iam-24gsw] pod update took 14.226546ms I0325 11:04:02.215795 3561 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-24gsw] addLogicalPort annotation time took 14.262445ms I0325 11:04:02.217332 3561 pods.go:271] [tsf-keycloak/tsf-iam-24gsw] addLogicalPort took 15.935688ms, libovsdb time 1.0934ms I0325 11:04:02.573111 3561 cni.go:403] [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default] I0325 11:04:02.584284 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-24gsw, hostIfaceName: 9f407385c519eb4, network: default, NAD default, SandboxID: "9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116", PCI device ID: , UID: "b7d8259b-ec47-48de-8905-a29b0d985381", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0325 11:04:02.632165 3561 cni.go:424] [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:45:71:af:59:c2\",\"name\":\"9f407385c519eb4\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/11617a89-1ad2-4f2d-80a2-294a395e5c9f\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:04:04.002028 3561 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0325 11:04:04.002060 3561 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0325 11:04:04.002069 3561 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0325 11:04:12.997394 3561 cni.go:403] [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default] DEL starting CNI request [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default] I0325 11:04:13.073914 3561 cni.go:424] [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default] DEL finished CNI request [tsf/test-tsf-iam f069a8ec72ef545d3351035cb5b5403eddc94fe83f899b757dd6f71c3db070da network default NAD default], result "{}", err I0325 11:04:13.133475 3561 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:04:13.133596 3561 pods.go:176] Deleting pod: tsf/test-tsf-iam I0325 11:04:13.142104 3561 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.42 I0325 11:04:13.142213 3561 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:04:22.051555 3561 cni.go:403] [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default] I0325 11:04:22.115260 3561 cni.go:424] [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-24gsw 9f407385c519eb4834c00c6fac5da5c558a5b1911a0692830050a45d73c58116 network default NAD default], result "{}", err I0325 11:04:22.163840 3561 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-24gsw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:04:22.163968 3561 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-24gsw I0325 11:04:22.165423 3561 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-24gsw, ips: 10.128.0.45 I0325 11:04:22.165454 3561 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-24gsw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:04:23.190564 3561 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-24gsw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:04:23.190587 3561 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-24gsw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:04:23.421663 3561 cni.go:403] [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default] I0325 11:04:23.498859 3561 cni.go:424] [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 01198b9cd74857e921b2f6bcc4e52d3ef1f439740cd14011f60ab4d00a10beb5 network default NAD default], result "{}", err I0325 11:04:23.547089 3561 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:04:23.547145 3561 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0325 11:04:23.548465 3561 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.44 I0325 11:04:23.548562 3561 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:04:24.056774 3561 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:04:24.056803 3561 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:04:24.062011 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0325 11:04:24.062039 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0325 11:04:24.088803 3561 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:04:24.088915 3561 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0325 11:04:24.106261 3561 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 17.381813ms I0325 11:04:24.106312 3561 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 17.435239ms I0325 11:04:24.107641 3561 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 18.85629ms, libovsdb time 965.232µs I0325 11:04:24.451257 3561 cni.go:403] [tsf-keycloak/keycloak-0 8564aa66c6fb7897bd96ae7bb3e51bc81c7c5b8cee381aa9e1930d1ab8c0f06f network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 8564aa66c6fb7897bd96ae7bb3e51bc81c7c5b8cee381aa9e1930d1ab8c0f06f network default NAD default] I0325 11:04:24.467288 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 8564aa66c6fb789, network: default, NAD default, SandboxID: "8564aa66c6fb7897bd96ae7bb3e51bc81c7c5b8cee381aa9e1930d1ab8c0f06f", PCI device ID: , UID: "adb92dde-c798-4a57-ba1f-dd2d06f896e0", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0325 11:04:24.522092 3561 cni.go:424] [tsf-keycloak/keycloak-0 8564aa66c6fb7897bd96ae7bb3e51bc81c7c5b8cee381aa9e1930d1ab8c0f06f network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 8564aa66c6fb7897bd96ae7bb3e51bc81c7c5b8cee381aa9e1930d1ab8c0f06f network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:e8:8b:6a:31:36\",\"name\":\"8564aa66c6fb789\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2cc74b5-ee36-4524-b2d3-680b1eeb698a\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:04:26.081786 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0325 11:04:26.082807 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:04:26.082893 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak I0325 11:04:27.943328 3561 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-pn85j for pod on switch ip-10-0-0-91.ec2.internal I0325 11:04:27.943487 3561 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j I0325 11:04:27.966402 3561 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j] pod update took 22.94584ms I0325 11:04:27.966508 3561 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j] addLogicalPort annotation time took 23.059638ms I0325 11:04:27.969431 3561 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j] addLogicalPort took 26.121993ms, libovsdb time 2.313823ms I0325 11:04:28.356678 3561 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j 70e6e227c715934bbe75d77d3a75779f97746335da6d494cd960de0b849b74a4 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j 70e6e227c715934bbe75d77d3a75779f97746335da6d494cd960de0b849b74a4 network default NAD default] I0325 11:04:28.371076 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-pn85j, hostIfaceName: 70e6e227c715934, network: default, NAD default, SandboxID: "70e6e227c715934bbe75d77d3a75779f97746335da6d494cd960de0b849b74a4", PCI device ID: , UID: "5baec996-d995-4495-9c97-ee61cbfc0e5d", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0325 11:04:28.479877 3561 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j 70e6e227c715934bbe75d77d3a75779f97746335da6d494cd960de0b849b74a4 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-pn85j 70e6e227c715934bbe75d77d3a75779f97746335da6d494cd960de0b849b74a4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:ef:dd:d7:2b:af\",\"name\":\"70e6e227c715934\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc8a20eb-5318-4d55-ad50-9f11c19ef8f7\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:04:32.057539 3561 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0325 11:04:32.058539 3561 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0325 11:04:33.145787 3561 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0325 11:04:34.539334 3561 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-56.ec2.internal I0325 11:04:34.539457 3561 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-56.ec2.internal: 131.396µs I0325 11:04:34.539598 3561 egressfirewall.go:628] Syncing node "ip-10-0-1-56.ec2.internal" for egress firewall W0325 11:04:41.801326 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0325 11:04:41.801457 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:04:41.801530 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0325 11:04:44.157441 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0325 11:04:45.220705 3561 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-djlcm for pod on switch ip-10-0-0-91.ec2.internal I0325 11:04:45.220904 3561 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm I0325 11:04:45.233965 3561 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm] pod update took 13.094618ms I0325 11:04:45.233996 3561 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm] addLogicalPort annotation time took 13.129431ms I0325 11:04:45.235755 3561 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm] addLogicalPort took 15.072479ms, libovsdb time 1.213058ms I0325 11:04:45.585811 3561 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm e376a2ce17fd4b7468b9313060ed37d1bb4be8a0431f022cd2f975da2c9d310a network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm e376a2ce17fd4b7468b9313060ed37d1bb4be8a0431f022cd2f975da2c9d310a network default NAD default] I0325 11:04:45.594618 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-djlcm, hostIfaceName: e376a2ce17fd4b7, network: default, NAD default, SandboxID: "e376a2ce17fd4b7468b9313060ed37d1bb4be8a0431f022cd2f975da2c9d310a", PCI device ID: , UID: "be0ec960-5f9e-4e0b-9b01-c41c1ec2b4ff", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0325 11:04:45.642388 3561 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm e376a2ce17fd4b7468b9313060ed37d1bb4be8a0431f022cd2f975da2c9d310a network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-djlcm e376a2ce17fd4b7468b9313060ed37d1bb4be8a0431f022cd2f975da2c9d310a network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:0f:55:b6:c6:55\",\"name\":\"e376a2ce17fd4b7\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68125ae4-35c3-4d10-80dc-19499d7515da\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:04:45.818157 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=49 I0325 11:04:56.882480 3561 namespace.go:142] [openshift-pipelines] updating namespace I0325 11:05:13.252125 3561 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:05:13.252155 3561 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:05:13.258195 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0325 11:05:13.258218 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0325 11:05:13.385694 3561 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-0-91.ec2.internal I0325 11:05:13.385887 3561 kube.go:256] Updating pod tsf/test-tsf-iam I0325 11:05:13.398297 3561 pod.go:62] [tsf/test-tsf-iam] pod update took 12.443242ms I0325 11:05:13.398331 3561 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 12.483101ms I0325 11:05:13.399961 3561 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 14.290891ms, libovsdb time 964.812µs I0325 11:05:13.748733 3561 cni.go:403] [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default] ADD starting CNI request [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default] I0325 11:05:13.757860 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: f43d58bef9863da, network: default, NAD default, SandboxID: "f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9", PCI device ID: , UID: "7b548e00-3fce-4944-98a5-89c10778f1f9", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0325 11:05:13.805595 3561 cni.go:424] [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default] ADD finished CNI request [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:0f:3e:3b:3e:34\",\"name\":\"f43d58bef9863da\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ccb4082f-df3a-4fb0-982d-580cf3372a7b\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:05:22.434832 3561 cni.go:403] [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default] DEL starting CNI request [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default] I0325 11:05:22.503660 3561 cni.go:424] [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default] DEL finished CNI request [tsf/test-tsf-iam f43d58bef9863da185d7fe47d15f940dc4579a7c937a565defe4aecebd20b6f9 network default NAD default], result "{}", err I0325 11:05:22.555222 3561 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:05:22.555322 3561 pods.go:176] Deleting pod: tsf/test-tsf-iam I0325 11:05:22.556834 3561 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.49 I0325 11:05:22.556869 3561 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:05:22.575261 3561 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:05:22.575335 3561 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:05:22.578825 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0325 11:05:22.578853 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0325 11:05:23.364242 3561 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-0-91.ec2.internal I0325 11:05:23.364379 3561 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0325 11:05:23.376592 3561 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 12.23432ms I0325 11:05:23.376615 3561 base_network_controller_pods.go:937] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 12.261157ms I0325 11:05:23.378373 3561 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 14.147231ms, libovsdb time 1.132073ms I0325 11:05:23.738106 3561 cni.go:403] [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default] ADD starting CNI request [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default] I0325 11:05:23.746923 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: 49932bbd0b6a127, network: default, NAD default, SandboxID: "49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83", PCI device ID: , UID: "7dbfee39-e136-46cf-872c-59b834e2dfe1", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0325 11:05:23.795094 3561 cni.go:424] [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default] ADD finished CNI request [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:2c:56:96:22:86\",\"name\":\"49932bbd0b6a127\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9db3ff2f-550a-45a6-bc35-8547d63f4289\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:05:25.197739 3561 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-kjz56-97qr7] creating logical port tsf-tas_ctlog-createtree-job-kjz56-97qr7 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:05:25.197879 3561 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-kjz56-97qr7 I0325 11:05:25.212175 3561 pod.go:62] [tsf-tas/ctlog-createtree-job-kjz56-97qr7] pod update took 14.33233ms I0325 11:05:25.212258 3561 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-kjz56-97qr7] addLogicalPort annotation time took 14.417715ms I0325 11:05:25.214902 3561 pods.go:271] [tsf-tas/ctlog-createtree-job-kjz56-97qr7] addLogicalPort took 17.180799ms, libovsdb time 1.249901ms I0325 11:05:25.559951 3561 cni.go:403] [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default] I0325 11:05:25.569284 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-kjz56-97qr7, hostIfaceName: f05519fc308049c, network: default, NAD default, SandboxID: "f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632", PCI device ID: , UID: "f3888cb5-b1b8-4e45-a367-d3195156b52d", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0325 11:05:25.615490 3561 cni.go:424] [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:27:52:38:29:52\",\"name\":\"f05519fc308049c\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d22547a-5abf-48b2-8983-d458fdb516a0\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:05:27.358010 3561 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0325 11:05:27.358034 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0325 11:05:33.399474 3561 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0325 11:05:36.473868 3561 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0325 11:05:48.554162 3561 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0325 11:05:49.697843 3561 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-2tgtw-g4s6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:05:49.698410 3561 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-2tgtw-g4s6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:05:50.594151 3561 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-2tgtw-g4s6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:05:50.594175 3561 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-2tgtw-g4s6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:37.880638 3561 cni.go:403] [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default] DEL starting CNI request [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default] I0325 11:06:37.946213 3561 cni.go:424] [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default] DEL finished CNI request [tsf-tas/test-tsf-tas 49932bbd0b6a1271ce1821bfc992e2c6cf03d9aae79da251a6085d1c25282e83 network default NAD default], result "{}", err I0325 11:06:37.995270 3561 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:37.995311 3561 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0325 11:06:37.996812 3561 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.50 I0325 11:06:37.996849 3561 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:38.033144 3561 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:38.033170 3561 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:38.040131 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0325 11:06:38.040155 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0325 11:06:38.695405 3561 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-85gts] creating logical port tsf_patch-tekton-config-85gts for pod on switch ip-10-0-0-91.ec2.internal I0325 11:06:38.695560 3561 kube.go:256] Updating pod tsf/patch-tekton-config-85gts I0325 11:06:38.707465 3561 pod.go:62] [tsf/patch-tekton-config-85gts] pod update took 11.940329ms I0325 11:06:38.707491 3561 base_network_controller_pods.go:937] [default/tsf/patch-tekton-config-85gts] addLogicalPort annotation time took 11.971758ms I0325 11:06:38.709066 3561 pods.go:271] [tsf/patch-tekton-config-85gts] addLogicalPort took 13.689839ms, libovsdb time 1.068821ms I0325 11:06:39.057406 3561 cni.go:403] [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default] ADD starting CNI request [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default] I0325 11:06:39.066417 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-85gts, hostIfaceName: 5c2002e226cd10c, network: default, NAD default, SandboxID: "5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466", PCI device ID: , UID: "105d8676-8e1b-46a4-a9b5-486c9e9f21b6", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0325 11:06:39.112946 3561 cni.go:424] [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default] ADD finished CNI request [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:8d:22:a4:f8:49\",\"name\":\"5c2002e226cd10c\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f620a41e-8b1a-43e1-9f16-0ec27fc8ba72\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:06:41.934484 3561 cni.go:403] [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default] I0325 11:06:42.018847 3561 cni.go:424] [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-kjz56-97qr7 f05519fc308049c9aae9d3e012acc56af5324a51bae72bfb038b5c0ab7a3c632 network default NAD default], result "{}", err I0325 11:06:42.992798 3561 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-gs4ft-252dc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:42.993471 3561 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-gs4ft-252dc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:43.873531 3561 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-gs4ft-252dc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:43.873561 3561 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-gs4ft-252dc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:44.516771 3561 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-kjz56-97qr7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:44.516815 3561 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-kjz56-97qr7 I0325 11:06:44.518583 3561 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-kjz56-97qr7, ips: 10.128.0.51 I0325 11:06:44.518621 3561 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-kjz56-97qr7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:44.925636 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0325 11:06:45.539419 3561 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-kjz56-97qr7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:45.539527 3561 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-kjz56-97qr7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:46.688499 3561 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-86876d4ccc-q5tqj] creating logical port tsf-tas_rekor-redis-86876d4ccc-q5tqj for pod on switch ip-10-0-0-91.ec2.internal I0325 11:06:46.688624 3561 kube.go:256] Updating pod tsf-tas/rekor-redis-86876d4ccc-q5tqj I0325 11:06:46.703008 3561 pod.go:62] [tsf-tas/rekor-redis-86876d4ccc-q5tqj] pod update took 14.423864ms I0325 11:06:46.703032 3561 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-86876d4ccc-q5tqj] addLogicalPort annotation time took 14.453444ms I0325 11:06:46.704501 3561 pods.go:271] [tsf-tas/rekor-redis-86876d4ccc-q5tqj] addLogicalPort took 16.025915ms, libovsdb time 1.016761ms I0325 11:06:46.822245 3561 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-5f47759896-zcmdh] creating logical port tsf-tas_rekor-search-ui-5f47759896-zcmdh for pod on switch ip-10-0-0-91.ec2.internal I0325 11:06:46.822328 3561 kube.go:256] Updating pod tsf-tas/rekor-search-ui-5f47759896-zcmdh I0325 11:06:46.838218 3561 pod.go:62] [tsf-tas/rekor-search-ui-5f47759896-zcmdh] pod update took 15.911897ms I0325 11:06:46.838301 3561 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-5f47759896-zcmdh] addLogicalPort annotation time took 15.99729ms I0325 11:06:46.839494 3561 pods.go:271] [tsf-tas/rekor-search-ui-5f47759896-zcmdh] addLogicalPort took 17.265011ms, libovsdb time 814.455µs I0325 11:06:46.959264 3561 cni.go:403] [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default] DEL starting CNI request [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default] I0325 11:06:47.045232 3561 cni.go:424] [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default] DEL finished CNI request [tsf/patch-tekton-config-85gts 5c2002e226cd10c3922cb4d115de336f44e8f4f703fe2ad78a7070f412a6a466 network default NAD default], result "{}", err I0325 11:06:47.096059 3561 obj_retry.go:464] Detected object tsf/patch-tekton-config-85gts of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:47.096164 3561 pods.go:176] Deleting pod: tsf/patch-tekton-config-85gts I0325 11:06:47.097695 3561 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-85gts, ips: 10.128.0.52 I0325 11:06:47.097735 3561 obj_retry.go:464] Detected object tsf/patch-tekton-config-85gts of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:47.181951 3561 cni.go:403] [tsf-tas/rekor-search-ui-5f47759896-zcmdh a4dda2aab41f2ae47a8d715d2346c215b492e3b249a5d354c24e8606253132fc network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-5f47759896-zcmdh a4dda2aab41f2ae47a8d715d2346c215b492e3b249a5d354c24e8606253132fc network default NAD default] I0325 11:06:47.191068 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-5f47759896-zcmdh, hostIfaceName: a4dda2aab41f2ae, network: default, NAD default, SandboxID: "a4dda2aab41f2ae47a8d715d2346c215b492e3b249a5d354c24e8606253132fc", PCI device ID: , UID: "dd4e1d90-6937-4d5a-810a-0e9ffeee6eb8", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0325 11:06:47.235819 3561 cni.go:424] [tsf-tas/rekor-search-ui-5f47759896-zcmdh a4dda2aab41f2ae47a8d715d2346c215b492e3b249a5d354c24e8606253132fc network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-5f47759896-zcmdh a4dda2aab41f2ae47a8d715d2346c215b492e3b249a5d354c24e8606253132fc network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:75:65:d7:85:ab\",\"name\":\"a4dda2aab41f2ae\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1e1902de-c08f-4810-bf74-ef75745acc64\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:06:47.661422 3561 cni.go:403] [tsf-tas/rekor-redis-86876d4ccc-q5tqj cf85a8955bad5b4354ec5a3b2471b991598baaf6164ce669ce57496b4339f53c network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-86876d4ccc-q5tqj cf85a8955bad5b4354ec5a3b2471b991598baaf6164ce669ce57496b4339f53c network default NAD default] I0325 11:06:47.671552 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-86876d4ccc-q5tqj, hostIfaceName: cf85a8955bad5b4, network: default, NAD default, SandboxID: "cf85a8955bad5b4354ec5a3b2471b991598baaf6164ce669ce57496b4339f53c", PCI device ID: , UID: "6187aa16-00ad-4afe-8b9a-b1fdffd79751", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0325 11:06:47.718861 3561 cni.go:424] [tsf-tas/rekor-redis-86876d4ccc-q5tqj cf85a8955bad5b4354ec5a3b2471b991598baaf6164ce669ce57496b4339f53c network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-86876d4ccc-q5tqj cf85a8955bad5b4354ec5a3b2471b991598baaf6164ce669ce57496b4339f53c network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:6d:37:72:70:1d\",\"name\":\"cf85a8955bad5b4\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c9b8dace-2549-4548-a0db-7ca4baf0bcbe\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:06:48.117113 3561 obj_retry.go:458] Detected object tsf/patch-tekton-config-85gts of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:48.117145 3561 obj_retry.go:458] Detected object tsf/patch-tekton-config-85gts of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:49.712365 3561 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-6c57f6494-59rzv] creating logical port tsf-tas_ctlog-6c57f6494-59rzv for pod on switch ip-10-0-0-91.ec2.internal I0325 11:06:49.712470 3561 kube.go:256] Updating pod tsf-tas/ctlog-6c57f6494-59rzv I0325 11:06:49.726267 3561 pod.go:62] [tsf-tas/ctlog-6c57f6494-59rzv] pod update took 13.830242ms I0325 11:06:49.726395 3561 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-6c57f6494-59rzv] addLogicalPort annotation time took 13.959735ms I0325 11:06:49.728509 3561 pods.go:271] [tsf-tas/ctlog-6c57f6494-59rzv] addLogicalPort took 16.163999ms, libovsdb time 1.519492ms I0325 11:06:50.307331 3561 cni.go:403] [tsf-tas/ctlog-6c57f6494-59rzv 1b11fefcd08caebee00bd3631726cb60f471d9dc9f8795b520691e5ad8901ed6 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-6c57f6494-59rzv 1b11fefcd08caebee00bd3631726cb60f471d9dc9f8795b520691e5ad8901ed6 network default NAD default] I0325 11:06:50.316643 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-6c57f6494-59rzv, hostIfaceName: 1b11fefcd08caeb, network: default, NAD default, SandboxID: "1b11fefcd08caebee00bd3631726cb60f471d9dc9f8795b520691e5ad8901ed6", PCI device ID: , UID: "d7a0136c-0ccf-4eaf-8c18-a31bbf8c0219", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0325 11:06:50.369414 3561 cni.go:424] [tsf-tas/ctlog-6c57f6494-59rzv 1b11fefcd08caebee00bd3631726cb60f471d9dc9f8795b520691e5ad8901ed6 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-6c57f6494-59rzv 1b11fefcd08caebee00bd3631726cb60f471d9dc9f8795b520691e5ad8901ed6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:f1:1d:ed:99:26\",\"name\":\"1b11fefcd08caeb\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2e9d89b-e4e8-419d-985e-7183331cbe4d\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:06:50.995860 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0325 11:06:51.022233 3561 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0325 11:06:52.027383 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0325 11:06:54.971866 3561 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-78mcp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:54.972425 3561 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-78mcp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:06:55.992338 3561 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-78mcp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:55.992403 3561 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-78mcp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:56.025619 3561 obj_retry.go:458] Detected object tsf/patch-tekton-config-85gts of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:56.025649 3561 obj_retry.go:458] Detected object tsf/patch-tekton-config-85gts of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:06:56.029656 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-85gts I0325 11:06:56.029681 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-85gts W0325 11:06:56.938420 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0325 11:07:01.828105 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=497 W0325 11:07:03.025066 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog W0325 11:07:03.053790 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis W0325 11:07:03.103688 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0325 11:07:08.151921 3561 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:08.152508 3561 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:08.169146 3561 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:08.169166 3561 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:08.175517 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0325 11:07:08.175536 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0325 11:07:08.734327 3561 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-0-91.ec2.internal I0325 11:07:08.734461 3561 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0325 11:07:08.747633 3561 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 13.192725ms I0325 11:07:08.747719 3561 base_network_controller_pods.go:937] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 13.28502ms I0325 11:07:08.749511 3561 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 15.206539ms, libovsdb time 1.097014ms I0325 11:07:09.099006 3561 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default] I0325 11:07:09.107628 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 72b435719c47371, network: default, NAD default, SandboxID: "72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723", PCI device ID: , UID: "43596826-ced7-4512-a5a7-9fab9052ec35", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0325 11:07:09.154742 3561 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:79:bd:1f:1d:0d\",\"name\":\"72b435719c47371\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a60bbe65-c8dc-4e45-92ef-593e73f470c3\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:07:12.137562 3561 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default] I0325 11:07:12.208250 3561 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 72b435719c4737188155e9743b66339b14ee09ff22583afb63d8698f09c7e723 network default NAD default], result "{}", err I0325 11:07:12.257498 3561 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:12.257531 3561 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0325 11:07:12.259304 3561 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.56 I0325 11:07:12.259335 3561 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:12.276774 3561 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:12.276852 3561 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:12.283227 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0325 11:07:12.283255 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0325 11:07:12.484564 3561 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-0-91.ec2.internal I0325 11:07:12.484658 3561 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0325 11:07:12.497689 3561 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.056084ms I0325 11:07:12.497716 3561 base_network_controller_pods.go:937] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.086595ms I0325 11:07:12.499409 3561 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 14.866264ms, libovsdb time 1.210875ms I0325 11:07:12.845965 3561 cni.go:403] [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default] ADD starting CNI request [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default] I0325 11:07:12.856740 3561 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 766c4357ef6c8ce, network: default, NAD default, SandboxID: "766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3", PCI device ID: , UID: "d3decd8a-03da-4cc8-8bf1-8d3b82b8a40f", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0325 11:07:12.907007 3561 cni.go:424] [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:34:14:50:c1:68\",\"name\":\"766c4357ef6c8ce\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc51cfc0-d75b-4e66-8647-3d3c10e39f53\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:07:15.240104 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0325 11:07:18.232316 3561 obj_retry.go:464] Detected object tsf-tpa/create-db-bwg5h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:18.232927 3561 obj_retry.go:464] Detected object tsf-tpa/create-db-bwg5h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:19.255175 3561 obj_retry.go:458] Detected object tsf-tpa/create-db-bwg5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:19.255233 3561 obj_retry.go:458] Detected object tsf-tpa/create-db-bwg5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:22.245501 3561 obj_retry.go:464] Detected object tsf-tpa/migrate-db-k6k2s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:22.246152 3561 obj_retry.go:464] Detected object tsf-tpa/migrate-db-k6k2s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:22.387293 3561 obj_retry.go:458] Detected object tsf-tpa/migrate-db-k6k2s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:22.387316 3561 obj_retry.go:458] Detected object tsf-tpa/migrate-db-k6k2s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:27.165520 3561 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-xmghm-dlvh9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:27.166119 3561 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-xmghm-dlvh9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:28.045369 3561 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-xmghm-dlvh9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:28.045450 3561 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-xmghm-dlvh9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0325 11:07:36.083439 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0325 11:07:37.200370 3561 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0325 11:07:38.230554 3561 obj_retry.go:464] Detected object tsf-tpa/create-importers-qbdvb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:38.231147 3561 obj_retry.go:464] Detected object tsf-tpa/create-importers-qbdvb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:07:39.252245 3561 obj_retry.go:458] Detected object tsf-tpa/create-importers-qbdvb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:39.252274 3561 obj_retry.go:458] Detected object tsf-tpa/create-importers-qbdvb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:07:40.143415 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0325 11:08:05.233206 3561 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0325 11:08:05.470497 3561 cni.go:403] [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default] DEL starting CNI request [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default] I0325 11:08:05.546521 3561 cni.go:424] [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 766c4357ef6c8ceb04132d8a61b218be2dc3fe81e5b0086c9dcc7eb5bcc950c3 network default NAD default], result "{}", err I0325 11:08:05.598208 3561 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:08:05.598251 3561 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0325 11:08:05.599705 3561 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.57 I0325 11:08:05.599741 3561 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:08:05.619225 3561 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:08:05.619248 3561 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:08:05.626151 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0325 11:08:05.626245 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0325 11:08:06.332184 3561 namespace.go:98] [build-service] adding namespace I0325 11:08:06.333135 3561 namespace.go:102] [build-service] adding namespace took 923.329µs I0325 11:08:06.336122 3561 namespace.go:98] [image-controller] adding namespace I0325 11:08:06.336584 3561 namespace.go:98] [integration-service] adding namespace I0325 11:08:06.337057 3561 namespace.go:102] [image-controller] adding namespace took 869.026µs I0325 11:08:06.337780 3561 namespace.go:102] [integration-service] adding namespace took 1.144044ms I0325 11:08:06.362157 3561 namespace.go:142] [build-service] updating namespace I0325 11:08:06.385758 3561 namespace.go:142] [build-service] updating namespace I0325 11:08:06.404580 3561 namespace.go:142] [image-controller] updating namespace I0325 11:08:06.451173 3561 namespace.go:142] [image-controller] updating namespace I0325 11:08:06.491853 3561 namespace.go:142] [integration-service] updating namespace I0325 11:08:06.509826 3561 namespace.go:142] [integration-service] updating namespace I0325 11:08:06.559810 3561 namespace.go:142] [build-service] updating namespace I0325 11:08:06.568802 3561 namespace.go:142] [integration-service] updating namespace I0325 11:08:06.597615 3561 namespace.go:98] [release-service] adding namespace I0325 11:08:06.598461 3561 namespace.go:102] [release-service] adding namespace took 801.953µs I0325 11:08:06.620757 3561 namespace.go:142] [release-service] updating namespace I0325 11:08:06.655269 3561 namespace.go:142] [release-service] updating namespace I0325 11:08:06.715994 3561 namespace.go:98] [konflux-info] adding namespace I0325 11:08:06.715994 3561 namespace.go:142] [konflux-ui] updating namespace I0325 11:08:06.716786 3561 namespace.go:102] [konflux-info] adding namespace took 711.196µs I0325 11:08:06.753827 3561 namespace.go:98] [namespace-lister] adding namespace I0325 11:08:06.754531 3561 namespace.go:102] [namespace-lister] adding namespace took 685.665µs I0325 11:08:06.788844 3561 namespace.go:142] [konflux-info] updating namespace I0325 11:08:06.803762 3561 namespace.go:142] [image-controller] updating namespace I0325 11:08:06.823260 3561 namespace.go:98] [enterprise-contract-service] adding namespace I0325 11:08:06.824263 3561 namespace.go:102] [enterprise-contract-service] adding namespace took 969.547µs I0325 11:08:06.867057 3561 namespace.go:142] [konflux-info] updating namespace I0325 11:08:06.890017 3561 namespace.go:142] [namespace-lister] updating namespace I0325 11:08:06.935877 3561 namespace.go:98] [default-tenant] adding namespace I0325 11:08:06.936782 3561 namespace.go:102] [default-tenant] adding namespace took 803.53µs I0325 11:08:06.945961 3561 namespace.go:98] [segment-bridge] adding namespace I0325 11:08:06.946748 3561 namespace.go:102] [segment-bridge] adding namespace took 769.476µs I0325 11:08:06.983349 3561 namespace.go:142] [namespace-lister] updating namespace I0325 11:08:07.025805 3561 namespace.go:142] [enterprise-contract-service] updating namespace I0325 11:08:07.087030 3561 namespace.go:142] [enterprise-contract-service] updating namespace I0325 11:08:07.124038 3561 namespace.go:142] [default-tenant] updating namespace I0325 11:08:07.177897 3561 namespace.go:142] [default-tenant] updating namespace I0325 11:08:07.203708 3561 namespace.go:142] [segment-bridge] updating namespace I0325 11:08:07.247754 3561 namespace.go:142] [segment-bridge] updating namespace I0325 11:08:07.602612 3561 namespace.go:142] [build-service] updating namespace I0325 11:08:07.658033 3561 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0325 11:08:07.658703 3561 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0325 11:08:07.658724 3561 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0325 11:08:07.661458 3561 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0325 11:08:07.682078 3561 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-schvg] creating logical port namespace-lister_namespace-lister-67655c4674-schvg for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:07.682197 3561 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-schvg I0325 11:08:07.695140 3561 pod.go:62] [namespace-lister/namespace-lister-67655c4674-schvg] pod update took 12.968376ms I0325 11:08:07.695168 3561 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-schvg] addLogicalPort annotation time took 13.001243ms I0325 11:08:07.696644 3561 pods.go:271] [namespace-lister/namespace-lister-67655c4674-schvg] addLogicalPort took 14.581601ms, libovsdb time 1.019915ms I0325 11:08:07.748483 3561 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0325 11:08:07.748509 3561 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0325 11:08:07.749731 3561 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0325 11:08:07.799097 3561 namespace.go:142] [default-tenant] updating namespace I0325 11:08:07.997537 3561 namespace.go:142] [enterprise-contract-service] updating namespace I0325 11:08:08.199550 3561 namespace.go:142] [image-controller] updating namespace I0325 11:08:08.401312 3561 namespace.go:142] [integration-service] updating namespace I0325 11:08:08.625101 3561 namespace.go:142] [konflux-info] updating namespace I0325 11:08:08.822835 3561 namespace.go:142] [namespace-lister] updating namespace I0325 11:08:09.017217 3561 namespace.go:142] [release-service] updating namespace I0325 11:08:09.162891 3561 base_network_controller_pods.go:487] [default/konflux-ui/dex-6cf664dc4d-9vvqb] creating logical port konflux-ui_dex-6cf664dc4d-9vvqb for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:09.163058 3561 kube.go:256] Updating pod konflux-ui/dex-6cf664dc4d-9vvqb I0325 11:08:09.176564 3561 pod.go:62] [konflux-ui/dex-6cf664dc4d-9vvqb] pod update took 13.531657ms I0325 11:08:09.176658 3561 base_network_controller_pods.go:937] [default/konflux-ui/dex-6cf664dc4d-9vvqb] addLogicalPort annotation time took 13.63158ms I0325 11:08:09.178731 3561 pods.go:271] [konflux-ui/dex-6cf664dc4d-9vvqb] addLogicalPort took 15.857481ms, libovsdb time 1.492902ms I0325 11:08:09.209001 3561 namespace.go:142] [segment-bridge] updating namespace I0325 11:08:09.324184 3561 base_network_controller_pods.go:487] [default/konflux-ui/proxy-6db99cc9b4-4fx4h] creating logical port konflux-ui_proxy-6db99cc9b4-4fx4h for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:09.324311 3561 kube.go:256] Updating pod konflux-ui/proxy-6db99cc9b4-4fx4h I0325 11:08:09.343191 3561 pod.go:62] [konflux-ui/proxy-6db99cc9b4-4fx4h] pod update took 18.94073ms I0325 11:08:09.343214 3561 base_network_controller_pods.go:937] [default/konflux-ui/proxy-6db99cc9b4-4fx4h] addLogicalPort annotation time took 18.970252ms I0325 11:08:09.344547 3561 pods.go:271] [konflux-ui/proxy-6db99cc9b4-4fx4h] addLogicalPort took 20.380709ms, libovsdb time 911.783µs I0325 11:08:09.345729 3561 namespace.go:142] [build-service] updating namespace I0325 11:08:09.366816 3561 namespace.go:142] [default-tenant] updating namespace I0325 11:08:09.384303 3561 namespace.go:142] [enterprise-contract-service] updating namespace I0325 11:08:09.405677 3561 namespace.go:142] [image-controller] updating namespace I0325 11:08:09.418706 3561 namespace.go:142] [integration-service] updating namespace I0325 11:08:09.438441 3561 namespace.go:142] [konflux-info] updating namespace I0325 11:08:09.449691 3561 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5] creating logical port image-controller_image-controller-controller-manager-7f574d4dd8-kz2h5 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:09.449866 3561 kube.go:256] Updating pod image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 I0325 11:08:09.459185 3561 namespace.go:142] [namespace-lister] updating namespace I0325 11:08:09.469429 3561 pod.go:62] [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5] pod update took 19.590349ms I0325 11:08:09.469451 3561 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5] addLogicalPort annotation time took 19.615452ms I0325 11:08:09.470778 3561 pods.go:271] [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5] addLogicalPort took 21.114248ms, libovsdb time 895.448µs I0325 11:08:09.479545 3561 namespace.go:142] [release-service] updating namespace I0325 11:08:09.504959 3561 namespace.go:142] [segment-bridge] updating namespace I0325 11:08:09.573592 3561 cni.go:403] [namespace-lister/namespace-lister-67655c4674-schvg bd28dc4804b9b06ea3358557fc24b4896ed621fd5608855a61bf3e04cbce2dfe network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-schvg bd28dc4804b9b06ea3358557fc24b4896ed621fd5608855a61bf3e04cbce2dfe network default NAD default] I0325 11:08:09.578313 3561 namespace.go:142] [build-service] updating namespace I0325 11:08:09.582643 3561 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-schvg, hostIfaceName: bd28dc4804b9b06, network: default, NAD default, SandboxID: "bd28dc4804b9b06ea3358557fc24b4896ed621fd5608855a61bf3e04cbce2dfe", PCI device ID: , UID: "e1e91f78-3767-45d4-9742-88ac7ac12fba", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0325 11:08:09.634046 3561 cni.go:424] [namespace-lister/namespace-lister-67655c4674-schvg bd28dc4804b9b06ea3358557fc24b4896ed621fd5608855a61bf3e04cbce2dfe network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-schvg bd28dc4804b9b06ea3358557fc24b4896ed621fd5608855a61bf3e04cbce2dfe network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:27:83:62:f4:ef\",\"name\":\"bd28dc4804b9b06\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bff9f807-236d-4498-b4e5-c200cc170141\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:08:09.653555 3561 namespace.go:142] [default-tenant] updating namespace I0325 11:08:09.719929 3561 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-xbt94] creating logical port integration-service_integration-service-controller-manager-759977d69f-xbt94 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:09.720089 3561 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-xbt94 I0325 11:08:09.733622 3561 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-xbt94] pod update took 13.560315ms I0325 11:08:09.733712 3561 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-xbt94] addLogicalPort annotation time took 13.654287ms I0325 11:08:09.735187 3561 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-xbt94] addLogicalPort took 15.279268ms, libovsdb time 932.423µs I0325 11:08:09.754059 3561 namespace.go:142] [enterprise-contract-service] updating namespace I0325 11:08:09.829102 3561 cni.go:403] [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default] I0325 11:08:09.830959 3561 namespace.go:142] [image-controller] updating namespace I0325 11:08:09.837974 3561 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-7f574d4dd8-kz2h5, hostIfaceName: 2b31f2f5751894c, network: default, NAD default, SandboxID: "2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa", PCI device ID: , UID: "e530aae3-4458-47b2-ae1b-e50b67463a7c", MAC: 0a:58:0a:80:00:3d, IPs: [10.128.0.61/23] I0325 11:08:09.884623 3561 cni.go:424] [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:c0:52:b1:e3:e0\",\"name\":\"2b31f2f5751894c\"},{\"mac\":\"0a:58:0a:80:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9bf80f40-1d1c-477e-9fb9-9392638f93cf\"}],\"ips\":[{\"address\":\"10.128.0.61/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:08:09.915322 3561 namespace.go:142] [integration-service] updating namespace I0325 11:08:09.995902 3561 namespace.go:142] [konflux-info] updating namespace I0325 11:08:10.067029 3561 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-zthr7] creating logical port build-service_build-service-controller-manager-7df75dd4df-zthr7 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:10.067243 3561 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-zthr7 I0325 11:08:10.077011 3561 namespace.go:142] [namespace-lister] updating namespace I0325 11:08:10.084857 3561 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-zthr7] pod update took 17.649269ms I0325 11:08:10.084963 3561 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-zthr7] addLogicalPort annotation time took 17.73225ms I0325 11:08:10.086368 3561 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-zthr7] addLogicalPort took 19.361861ms, libovsdb time 915.522µs I0325 11:08:10.161528 3561 namespace.go:142] [release-service] updating namespace I0325 11:08:10.264256 3561 namespace.go:142] [segment-bridge] updating namespace I0325 11:08:10.478465 3561 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-zthr7 a6220b3de2f31b299bdde9915999f785a6a82fa6e3879b6b93ea32eeb26be08e network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-zthr7 a6220b3de2f31b299bdde9915999f785a6a82fa6e3879b6b93ea32eeb26be08e network default NAD default] I0325 11:08:10.490225 3561 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-zthr7, hostIfaceName: a6220b3de2f31b2, network: default, NAD default, SandboxID: "a6220b3de2f31b299bdde9915999f785a6a82fa6e3879b6b93ea32eeb26be08e", PCI device ID: , UID: "30c40f02-c8ab-4508-98b3-a7b16fc39650", MAC: 0a:58:0a:80:00:3f, IPs: [10.128.0.63/23] I0325 11:08:10.549596 3561 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-zthr7 a6220b3de2f31b299bdde9915999f785a6a82fa6e3879b6b93ea32eeb26be08e network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-zthr7 a6220b3de2f31b299bdde9915999f785a6a82fa6e3879b6b93ea32eeb26be08e network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:d9:c0:9f:1d:51\",\"name\":\"a6220b3de2f31b2\"},{\"mac\":\"0a:58:0a:80:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a0f0955-8f30-44a3-b827-ec129d098cf2\"}],\"ips\":[{\"address\":\"10.128.0.63/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:08:11.009773 3561 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-vx6qv] creating logical port release-service_release-service-controller-manager-854f775895-vx6qv for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:11.010531 3561 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-vx6qv I0325 11:08:11.023092 3561 pod.go:62] [release-service/release-service-controller-manager-854f775895-vx6qv] pod update took 12.596506ms I0325 11:08:11.023183 3561 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-vx6qv] addLogicalPort annotation time took 12.692672ms I0325 11:08:11.025048 3561 pods.go:271] [release-service/release-service-controller-manager-854f775895-vx6qv] addLogicalPort took 15.293338ms, libovsdb time 1.280853ms W0325 11:08:11.527539 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0325 11:08:12.189965 3561 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-6989f8fcc5-swv6h] creating logical port integration-service_integration-service-controller-manager-6989f8fcc5-swv6h for pod on switch ip-10-0-0-91.ec2.internal I0325 11:08:12.190095 3561 kube.go:256] Updating pod integration-service/integration-service-controller-manager-6989f8fcc5-swv6h I0325 11:08:12.203345 3561 pod.go:62] [integration-service/integration-service-controller-manager-6989f8fcc5-swv6h] pod update took 13.282825ms I0325 11:08:12.203377 3561 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-6989f8fcc5-swv6h] addLogicalPort annotation time took 13.33158ms I0325 11:08:12.204914 3561 pods.go:271] [integration-service/integration-service-controller-manager-6989f8fcc5-swv6h] addLogicalPort took 14.98961ms, libovsdb time 996.555µs W0325 11:08:12.529119 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0325 11:08:13.682107 3561 cni.go:403] [konflux-ui/dex-6cf664dc4d-9vvqb eee5f65c4a5f5c0cf8f8ed3606f07c5cd67cb91bd07a5eb43809040bb5e64020 network default NAD default] ADD starting CNI request [konflux-ui/dex-6cf664dc4d-9vvqb eee5f65c4a5f5c0cf8f8ed3606f07c5cd67cb91bd07a5eb43809040bb5e64020 network default NAD default] I0325 11:08:13.695567 3561 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-6cf664dc4d-9vvqb, hostIfaceName: eee5f65c4a5f5c0, network: default, NAD default, SandboxID: "eee5f65c4a5f5c0cf8f8ed3606f07c5cd67cb91bd07a5eb43809040bb5e64020", PCI device ID: , UID: "56ec1f14-444b-42f9-bf9e-bdcaf4317a16", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0325 11:08:13.747415 3561 cni.go:424] [konflux-ui/dex-6cf664dc4d-9vvqb eee5f65c4a5f5c0cf8f8ed3606f07c5cd67cb91bd07a5eb43809040bb5e64020 network default NAD default] ADD finished CNI request [konflux-ui/dex-6cf664dc4d-9vvqb eee5f65c4a5f5c0cf8f8ed3606f07c5cd67cb91bd07a5eb43809040bb5e64020 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:cd:28:86:f3:75\",\"name\":\"eee5f65c4a5f5c0\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/196660c4-238f-47b1-a679-d6f6f4fdb64f\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:08:14.069607 3561 cni.go:403] [integration-service/integration-service-controller-manager-6989f8fcc5-swv6h bfd2515a4a8034697bdbe101b2cdfc121ec6e78bda29e72f835a72a7126acab5 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-6989f8fcc5-swv6h bfd2515a4a8034697bdbe101b2cdfc121ec6e78bda29e72f835a72a7126acab5 network default NAD default] I0325 11:08:14.078592 3561 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-6989f8fcc5-swv6h, hostIfaceName: bfd2515a4a80346, network: default, NAD default, SandboxID: "bfd2515a4a8034697bdbe101b2cdfc121ec6e78bda29e72f835a72a7126acab5", PCI device ID: , UID: "bbe1cd2c-2334-4270-8452-6a0e63540f74", MAC: 0a:58:0a:80:00:41, IPs: [10.128.0.65/23] I0325 11:08:14.125773 3561 cni.go:424] [integration-service/integration-service-controller-manager-6989f8fcc5-swv6h bfd2515a4a8034697bdbe101b2cdfc121ec6e78bda29e72f835a72a7126acab5 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-6989f8fcc5-swv6h bfd2515a4a8034697bdbe101b2cdfc121ec6e78bda29e72f835a72a7126acab5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:4a:8a:f7:38:8a\",\"name\":\"bfd2515a4a80346\"},{\"mac\":\"0a:58:0a:80:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f16d87e5-522a-4cce-acf8-5386c42bfebd\"}],\"ips\":[{\"address\":\"10.128.0.65/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:08:14.551929 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0325 11:08:14.574273 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0325 11:08:15.567325 3561 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0325 11:08:16.116169 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0325 11:08:17.804278 3561 cni.go:403] [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default] ADD starting CNI request [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default] I0325 11:08:17.804988 3561 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default] I0325 11:08:17.817081 3561 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-6db99cc9b4-4fx4h, hostIfaceName: 9373fb7f42cd100, network: default, NAD default, SandboxID: "9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc", PCI device ID: , UID: "ad0c7112-9c40-4a59-a52b-f72a977418f2", MAC: 0a:58:0a:80:00:3c, IPs: [10.128.0.60/23] I0325 11:08:17.817125 3561 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-xbt94, hostIfaceName: ab6481cae2ed3ac, network: default, NAD default, SandboxID: "ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287", PCI device ID: , UID: "094f4525-95f6-4054-b8bf-14706e0ac188", MAC: 0a:58:0a:80:00:3e, IPs: [10.128.0.62/23] I0325 11:08:17.878189 3561 cni.go:424] [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default] ADD finished CNI request [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:ce:cd:47:a8:8c\",\"name\":\"9373fb7f42cd100\"},{\"mac\":\"0a:58:0a:80:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ab3b41b1-f532-415e-ab7a-6ef8c3518527\"}],\"ips\":[{\"address\":\"10.128.0.60/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:08:17.897013 3561 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:f0:a0:87:ac:f4\",\"name\":\"ab6481cae2ed3ac\"},{\"mac\":\"0a:58:0a:80:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/790f76fe-eb54-41c7-abdd-8117e70257cb\"}],\"ips\":[{\"address\":\"10.128.0.62/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0325 11:08:18.595469 3561 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0325 11:08:18.597489 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0325 11:08:18.638681 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0325 11:08:18.641051 3561 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0325 11:08:19.194699 3561 cni.go:403] [release-service/release-service-controller-manager-854f775895-vx6qv 422b0cbfd9fa11d1f1b7e907ee843ad624ec964302c2d0e8e1e3f9b71cb71d69 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-vx6qv 422b0cbfd9fa11d1f1b7e907ee843ad624ec964302c2d0e8e1e3f9b71cb71d69 network default NAD default] I0325 11:08:19.206182 3561 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-vx6qv, hostIfaceName: 422b0cbfd9fa11d, network: default, NAD default, SandboxID: "422b0cbfd9fa11d1f1b7e907ee843ad624ec964302c2d0e8e1e3f9b71cb71d69", PCI device ID: , UID: "5ef7b3ee-886a-4a79-b21a-072f28c8dee6", MAC: 0a:58:0a:80:00:40, IPs: [10.128.0.64/23] I0325 11:08:19.268611 3561 cni.go:424] [release-service/release-service-controller-manager-854f775895-vx6qv 422b0cbfd9fa11d1f1b7e907ee843ad624ec964302c2d0e8e1e3f9b71cb71d69 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-vx6qv 422b0cbfd9fa11d1f1b7e907ee843ad624ec964302c2d0e8e1e3f9b71cb71d69 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:d9:43:54:d5:4e\",\"name\":\"422b0cbfd9fa11d\"},{\"mac\":\"0a:58:0a:80:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/990866db-6b08-494c-895f-cb77a5c55795\"}],\"ips\":[{\"address\":\"10.128.0.64/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:08:20.376898 3561 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-8dx45 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:08:20.377582 3561 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-8dx45 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0325 11:08:21.376268 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0325 11:08:21.376288 3561 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0325 11:08:21.398201 3561 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-8dx45 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:08:21.398231 3561 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-8dx45 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0325 11:08:23.644554 3561 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0325 11:08:23.644666 3561 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0325 11:08:23.670902 3561 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0325 11:08:23.670918 3561 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0325 11:08:25.565209 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0325 11:08:25.589901 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0325 11:08:29.614998 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0325 11:08:29.616708 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0325 11:08:29.806477 3561 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default] I0325 11:08:29.880572 3561 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-xbt94 ab6481cae2ed3ac40ca92df8d3328de9fe3e9ce3bb9d02cb41477dc1ad435287 network default NAD default], result "{}", err I0325 11:08:29.944895 3561 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-xbt94 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:08:29.945013 3561 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-xbt94 I0325 11:08:29.946574 3561 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-xbt94, ips: 10.128.0.62 I0325 11:08:29.946609 3561 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-xbt94 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0325 11:08:30.621176 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex I0325 11:08:30.696228 3561 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-xbt94 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:08:30.696257 3561 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-xbt94 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:08:30.702641 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-xbt94 I0325 11:08:30.702717 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-xbt94 W0325 11:08:34.658568 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0325 11:08:34.659079 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0325 11:08:44.120318 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0325 11:08:45.819287 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=76 W0325 11:08:58.446226 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0325 11:08:58.446291 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0325 11:08:58.446309 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0325 11:08:58.446316 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0325 11:08:59.817486 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 W0325 11:09:01.720131 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0325 11:09:01.720152 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0325 11:09:17.825248 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=157 I0325 11:09:28.206036 3561 cni.go:403] [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default] DEL starting CNI request [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default] I0325 11:09:28.284579 3561 cni.go:424] [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default] DEL finished CNI request [konflux-ui/proxy-6db99cc9b4-4fx4h 9373fb7f42cd100c9f044d64a84632dc8abd5dc17a669730348250cc7b9e69cc network default NAD default], result "{}", err I0325 11:09:28.346601 3561 obj_retry.go:464] Detected object konflux-ui/proxy-6db99cc9b4-4fx4h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:09:28.346680 3561 pods.go:176] Deleting pod: konflux-ui/proxy-6db99cc9b4-4fx4h I0325 11:09:28.348075 3561 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-6db99cc9b4-4fx4h, ips: 10.128.0.60 I0325 11:09:28.348107 3561 obj_retry.go:464] Detected object konflux-ui/proxy-6db99cc9b4-4fx4h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:09:29.057856 3561 obj_retry.go:458] Detected object konflux-ui/proxy-6db99cc9b4-4fx4h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:09:29.057883 3561 obj_retry.go:458] Detected object konflux-ui/proxy-6db99cc9b4-4fx4h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:09:29.066008 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-6db99cc9b4-4fx4h I0325 11:09:29.066101 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-6db99cc9b4-4fx4h I0325 11:09:29.940418 3561 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:09:29.941038 3561 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:09:29.977088 3561 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:09:29.977113 3561 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:09:29.981994 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0325 11:09:29.982078 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0325 11:09:50.680140 3561 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-78bccb8d54-4lwg2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:09:50.680777 3561 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-78bccb8d54-4lwg2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:09:50.753264 3561 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-q7ghp] creating logical port image-controller_image-controller-controller-manager-56b684c87-q7ghp for pod on switch ip-10-0-0-91.ec2.internal I0325 11:09:50.753363 3561 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-q7ghp I0325 11:09:50.766854 3561 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-q7ghp] pod update took 13.520423ms I0325 11:09:50.766881 3561 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-q7ghp] addLogicalPort annotation time took 13.551908ms I0325 11:09:50.768564 3561 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-q7ghp] addLogicalPort took 15.314477ms, libovsdb time 1.14024ms I0325 11:09:50.921861 3561 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-78bccb8d54-4lwg2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:09:50.921972 3561 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-78bccb8d54-4lwg2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:09:50.927869 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-78bccb8d54-4lwg2 I0325 11:09:50.927893 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-78bccb8d54-4lwg2 I0325 11:09:51.121814 3561 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default] I0325 11:09:51.132324 3561 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-q7ghp, hostIfaceName: afc3fa1d3ae675f, network: default, NAD default, SandboxID: "afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a", PCI device ID: , UID: "28bcba90-7cef-4c5b-81a8-85286faceafa", MAC: 0a:58:0a:80:00:42, IPs: [10.128.0.66/23] I0325 11:09:51.182429 3561 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:c3:9d:53:76:5f\",\"name\":\"afc3fa1d3ae675f\"},{\"mac\":\"0a:58:0a:80:00:42\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b87c46d6-082f-40f6-a6dc-2a1519e23dba\"}],\"ips\":[{\"address\":\"10.128.0.66/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:10:00.185037 3561 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29573950-s4vc8 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:10:00.185163 3561 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 I0325 11:10:00.197827 3561 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8] pod update took 12.694252ms I0325 11:10:00.197856 3561 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8] addLogicalPort annotation time took 12.730379ms I0325 11:10:00.199628 3561 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8] addLogicalPort took 14.613752ms, libovsdb time 1.095556ms I0325 11:10:00.547313 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default] I0325 11:10:00.556132 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29573950-s4vc8, hostIfaceName: 6b3cac29713b18b, network: default, NAD default, SandboxID: "6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31", PCI device ID: , UID: "21f9673e-6b0f-4678-95bb-1974abff3110", MAC: 0a:58:0a:80:00:43, IPs: [10.128.0.67/23] I0325 11:10:00.812225 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:29:cd:75:17:ad\",\"name\":\"6b3cac29713b18b\"},{\"mac\":\"0a:58:0a:80:00:43\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7cb40e98-9680-4640-aabf-689be00251e7\"}],\"ips\":[{\"address\":\"10.128.0.67/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:10:04.294830 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default] I0325 11:10:04.367575 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 6b3cac29713b18bc97cceb272a0a24e932c06b86a447e4cb5701a962db44db31 network default NAD default], result "{}", err I0325 11:10:04.408722 3561 cni.go:403] [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default] I0325 11:10:04.421669 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:10:04.421716 3561 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 I0325 11:10:04.423220 3561 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8, ips: 10.128.0.67 I0325 11:10:04.423252 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:10:04.497197 3561 cni.go:424] [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 2b31f2f5751894cef1d76f3906483685e8d72f8167bc1defc26c73e9edd634aa network default NAD default], result "{}", err I0325 11:10:04.504355 3561 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl] creating logical port konflux-operator_konflux-operator-controller-manager-78bccb8d54-lr6wl for pod on switch ip-10-0-0-91.ec2.internal I0325 11:10:04.504505 3561 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl I0325 11:10:04.519563 3561 pod.go:62] [konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl] pod update took 15.07605ms I0325 11:10:04.519591 3561 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl] addLogicalPort annotation time took 15.11524ms I0325 11:10:04.521399 3561 pods.go:271] [konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl] addLogicalPort took 17.058823ms, libovsdb time 1.110559ms I0325 11:10:04.559988 3561 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:10:04.560024 3561 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 I0325 11:10:04.561678 3561 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5, ips: 10.128.0.61 I0325 11:10:04.561712 3561 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:10:04.875250 3561 cni.go:403] [konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl 62b6da82ce5291dce365b8dc07783cfb4f33358acd9b6e9cac2edb116e3bc1fc network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl 62b6da82ce5291dce365b8dc07783cfb4f33358acd9b6e9cac2edb116e3bc1fc network default NAD default] I0325 11:10:04.884023 3561 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-78bccb8d54-lr6wl, hostIfaceName: 62b6da82ce5291d, network: default, NAD default, SandboxID: "62b6da82ce5291dce365b8dc07783cfb4f33358acd9b6e9cac2edb116e3bc1fc", PCI device ID: , UID: "759d03a9-ea95-4a43-8357-18244b8738ff", MAC: 0a:58:0a:80:00:44, IPs: [10.128.0.68/23] I0325 11:10:04.934297 3561 cni.go:424] [konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl 62b6da82ce5291dce365b8dc07783cfb4f33358acd9b6e9cac2edb116e3bc1fc network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-78bccb8d54-lr6wl 62b6da82ce5291dce365b8dc07783cfb4f33358acd9b6e9cac2edb116e3bc1fc network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:cf:90:59:0c:7e\",\"name\":\"62b6da82ce5291d\"},{\"mac\":\"0a:58:0a:80:00:44\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/34d5f5a8-5d65-4e49-ba91-0b0c08f3bb14\"}],\"ips\":[{\"address\":\"10.128.0.68/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:10:05.303101 3561 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:10:05.303132 3561 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:10:05.306718 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:10:05.306767 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573950-s4vc8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:10:05.311603 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 I0325 11:10:05.311697 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-7f574d4dd8-kz2h5 I0325 11:10:08.118745 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 W0325 11:10:08.336617 3561 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0325 11:10:19.335795 3561 util.go:959] Failed to build node endpoints for node ip-10-0-0-91.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0325 11:10:26.785642 3561 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl] creating logical port image-controller_image-controller-controller-manager-7f574d4dd8-m9cfl for pod on switch ip-10-0-0-91.ec2.internal I0325 11:10:26.785733 3561 kube.go:256] Updating pod image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl I0325 11:10:26.798242 3561 pod.go:62] [image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl] pod update took 12.531084ms I0325 11:10:26.798266 3561 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl] addLogicalPort annotation time took 12.561341ms I0325 11:10:26.799615 3561 pods.go:271] [image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl] addLogicalPort took 13.991799ms, libovsdb time 855.152µs I0325 11:10:27.157012 3561 cni.go:403] [image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl 50175c82223560fc73584b3eff918a14573f94a56665f0e8c3cd1ea4cb1b0c33 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl 50175c82223560fc73584b3eff918a14573f94a56665f0e8c3cd1ea4cb1b0c33 network default NAD default] I0325 11:10:27.168708 3561 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-7f574d4dd8-m9cfl, hostIfaceName: 50175c82223560f, network: default, NAD default, SandboxID: "50175c82223560fc73584b3eff918a14573f94a56665f0e8c3cd1ea4cb1b0c33", PCI device ID: , UID: "0af2dddb-cceb-4c56-a2b5-134968e1b0c1", MAC: 0a:58:0a:80:00:45, IPs: [10.128.0.69/23] I0325 11:10:27.219724 3561 cni.go:424] [image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl 50175c82223560fc73584b3eff918a14573f94a56665f0e8c3cd1ea4cb1b0c33 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-7f574d4dd8-m9cfl 50175c82223560fc73584b3eff918a14573f94a56665f0e8c3cd1ea4cb1b0c33 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:4f:a6:2e:87:23\",\"name\":\"50175c82223560f\"},{\"mac\":\"0a:58:0a:80:00:45\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ef5518e4-c652-4093-82ef-16ee85a6ebf0\"}],\"ips\":[{\"address\":\"10.128.0.69/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:10:29.822553 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=134 I0325 11:10:38.638085 3561 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default] I0325 11:10:38.716569 3561 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-56b684c87-q7ghp afc3fa1d3ae675f788176b98c42a75a443130dc3af0db9172cf10eb1e4efae7a network default NAD default], result "{}", err I0325 11:10:38.769949 3561 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-56b684c87-q7ghp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:10:38.770013 3561 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-56b684c87-q7ghp I0325 11:10:38.771503 3561 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-56b684c87-q7ghp, ips: 10.128.0.66 I0325 11:10:38.771538 3561 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-56b684c87-q7ghp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:10:39.505782 3561 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-56b684c87-q7ghp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:10:39.505851 3561 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-56b684c87-q7ghp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:10:39.511219 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-56b684c87-q7ghp I0325 11:10:39.511244 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-56b684c87-q7ghp I0325 11:11:36.282247 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0325 11:11:47.019727 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0325 11:12:03.928263 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0325 11:12:45.939149 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0325 11:14:21.821262 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=38 I0325 11:14:39.236604 3561 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0325 11:14:43.146165 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0325 11:15:35.243731 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0325 11:15:38.119251 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0325 11:15:39.830903 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=241 I0325 11:15:51.822126 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=40 W0325 11:15:58.894282 3561 duration.go:461] Config duration recorder: measurement expired for pod/openshift-console/console-586dd9bc44-q7tl2 I0325 11:17:02.828048 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0325 11:17:19.052214 3561 obj_retry.go:458] Detected object tsf-tpa/create-db-bwg5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:17:19.052258 3561 obj_retry.go:458] Detected object tsf-tpa/create-db-bwg5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:17:19.054685 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-bwg5h I0325 11:17:19.054742 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-bwg5h I0325 11:17:22.032818 3561 obj_retry.go:458] Detected object tsf-tpa/migrate-db-k6k2s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:17:22.032868 3561 obj_retry.go:458] Detected object tsf-tpa/migrate-db-k6k2s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:17:22.036027 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-k6k2s I0325 11:17:22.036048 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-k6k2s I0325 11:17:37.123525 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0325 11:17:39.031957 3561 obj_retry.go:458] Detected object tsf-tpa/create-importers-qbdvb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:17:39.032001 3561 obj_retry.go:458] Detected object tsf-tpa/create-importers-qbdvb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:17:39.035791 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-qbdvb I0325 11:17:39.035841 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-qbdvb I0325 11:17:41.825819 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=11 I0325 11:17:49.125657 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0325 11:18:05.820259 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0325 11:18:49.942600 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0325 11:18:51.931491 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0325 11:19:16.023101 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0325 11:20:00.208589 3561 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29573960-8w756 for pod on switch ip-10-0-0-91.ec2.internal I0325 11:20:00.208712 3561 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 I0325 11:20:00.235490 3561 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756] pod update took 26.802775ms I0325 11:20:00.235586 3561 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756] addLogicalPort annotation time took 26.904334ms I0325 11:20:00.237245 3561 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756] addLogicalPort took 28.670084ms, libovsdb time 1.011174ms I0325 11:20:00.577854 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default] I0325 11:20:00.586556 3561 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29573960-8w756, hostIfaceName: 82172d4b22d0793, network: default, NAD default, SandboxID: "82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04", PCI device ID: , UID: "fe597bb3-918b-4174-acd5-248f120dcdac", MAC: 0a:58:0a:80:00:46, IPs: [10.128.0.70/23] I0325 11:20:00.636212 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:c0:6d:30:36:c1\",\"name\":\"82172d4b22d0793\"},{\"mac\":\"0a:58:0a:80:00:46\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5d6da271-3fd6-4d71-a1de-8c7e9e109782\"}],\"ips\":[{\"address\":\"10.128.0.70/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0325 11:20:03.899472 3561 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default] I0325 11:20:03.978231 3561 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 82172d4b22d079327ac7dff9834c391b157d0510c1f220f55a825901ae9e3c04 network default NAD default], result "{}", err I0325 11:20:04.029968 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0325 11:20:04.030071 3561 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 I0325 11:20:04.031639 3561 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756, ips: 10.128.0.70 I0325 11:20:04.031670 3561 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0325 11:20:04.909164 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:20:04.909210 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573960-8w756 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:20:04.939311 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:20:04.939376 3561 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0325 11:20:04.941287 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s I0325 11:20:04.941311 3561 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29573930-vn68s I0325 11:20:07.239094 3561 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0325 11:21:05.149220 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0325 11:21:18.285627 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0325 11:21:51.246781 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0325 11:22:18.834114 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=24 I0325 11:22:44.824858 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0325 11:22:53.186404 3561 namespace.go:98] [default-tenant-managed] adding namespace I0325 11:22:53.187371 3561 namespace.go:102] [default-tenant-managed] adding namespace took 938.39µs I0325 11:22:53.249850 3561 namespace.go:142] [default-tenant-managed] updating namespace I0325 11:22:53.267962 3561 namespace.go:142] [default-tenant-managed] updating namespace I0325 11:22:53.828529 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0325 11:22:54.195577 3561 namespace.go:142] [default-tenant-managed] updating namespace I0325 11:22:54.327293 3561 namespace.go:142] [default-tenant-managed] updating namespace I0325 11:22:54.426200 3561 namespace.go:142] [default-tenant-managed] updating namespace I0325 11:23:30.126309 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0325 11:23:48.831257 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=12 I0325 11:24:41.128277 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0325 11:24:55.122323 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0325 11:25:07.825313 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=46 I0325 11:25:37.242421 3561 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0325 11:26:36.822926 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0325 11:26:43.289111 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0325 11:26:51.945861 3561 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0325 11:27:27.836974 3561 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=7 I0325 11:27:52.151806 3561 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6