+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-151.ec2.internal ++ [[ -n ip-10-0-2-151.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-151.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0326 23:32:35.393381895 - disable conntrack on geneve port' I0326 23:32:35.393381895 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0326 23:32:35.412014830 - starting ovnkube-node + echo 'I0326 23:32:35.412014830 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-151.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-151.ec2.internal --init-node ip-10-0-2-151.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-151.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0326 23:32:35.456114 3878 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0326 23:32:35.456172 3878 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-8ed5abe7ae.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0326 23:32:35.458172 3878 kube.go:419] Waiting for certificate I0326 23:32:35.458212 3878 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0326 23:32:35.458248 3878 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0326 23:32:35.459380 3878 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:32:35.487003 3878 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:35.487022 3878 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:35.489549 3878 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:35.489637 3878 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-cr2g9" I0326 23:32:35.495885 3878 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-cr2g9" I0326 23:32:35.495932 3878 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:32:36.458729 3878 kube.go:426] Certificate found I0326 23:32:36.459231 3878 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:32:36.460416 3878 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0326 23:32:36.499681 3878 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-27 23:27:35 +0000 UTC" deadline="2026-03-27 19:39:02.349785281 +0000 UTC" I0326 23:32:36.499714 3878 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h6m25.850080274s" I0326 23:32:36.519604 3878 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:32:36.533847 3878 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0326 23:32:36.534010 3878 factory.go:531] Starting watch factory I0326 23:32:36.534152 3878 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.534225 3878 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.534734 3878 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.534772 3878 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535479 3878 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535509 3878 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535531 3878 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535558 3878 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535564 3878 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535577 3878 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535538 3878 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.535757 3878 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.537916 3878 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0326 23:32:36.542028 3878 controller_manager.go:368] Starting the ovnkube controller I0326 23:32:36.542038 3878 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-151.ec2.internal I0326 23:32:36.542082 3878 controller_manager.go:393] NBDB zone sync took: 37.61µs I0326 23:32:36.542091 3878 factory.go:531] Starting watch factory I0326 23:32:36.555289 3878 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0326 23:32:36.555292 3878 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.555465 3878 ovn_northd.go:121] Registering OVN northd metrics I0326 23:32:36.555479 3878 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.556081 3878 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.560198 3878 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0326 23:32:36.560252 3878 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0326 23:32:36.560823 3878 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.562711 3878 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.564737 3878 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:32:36.577439 3878 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0326 23:32:36.639594 3878 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:32:36.639626 3878 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:32:36.639630 3878 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:32:36.639633 3878 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:32:36.639637 3878 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:32:36.639640 3878 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:32:36.639723 3878 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:36.639746 3878 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:36.639737 3878 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:36.639766 3878 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:36.642755 3878 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:36.642752 3878 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:32:36.644927 3878 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:32:36.644938 3878 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:32:36.644998 3878 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.645012 3878 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.647566 3878 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.648519 3878 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:32:36.648530 3878 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:32:36.648535 3878 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:32:36.648540 3878 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:32:36.648544 3878 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:32:36.648548 3878 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:32:36.650642 3878 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:32:36.651129 3878 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.651145 3878 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.653762 3878 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:32:36.653777 3878 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:32:36.653918 3878 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.655881 3878 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:32:36.655930 3878 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.655940 3878 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.658866 3878 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.658873 3878 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:32:36.660971 3878 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:32:36.661059 3878 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.661074 3878 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.663541 3878 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.664508 3878 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:32:36.666655 3878 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:32:36.666725 3878 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.666743 3878 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.669694 3878 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.669703 3878 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:32:36.672002 3878 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:32:36.672620 3878 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:32:36.672633 3878 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:32:36.674845 3878 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:32:36.674959 3878 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:32:36.678027 3878 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:32:36.678098 3878 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.678111 3878 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.678143 3878 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.678157 3878 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.680240 3878 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:32:36.680710 3878 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.680743 3878 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:32:36.683068 3878 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:32:36.683077 3878 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:32:36.683083 3878 factory.go:636] Watch Factory start up complete, took: 149.079601ms I0326 23:32:36.683150 3878 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0326 23:32:36.685492 3878 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:32:36.691016 3878 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:32:36.691025 3878 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:32:36.691032 3878 factory.go:636] Watch Factory start up complete, took: 148.940276ms I0326 23:32:36.691042 3878 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-151.ec2.internal" zone I0326 23:32:36.718128 3878 default_node_network_controller.go:709] Initializing the default node network controller I0326 23:32:36.773859 3878 udn_isolation.go:101] Starting UDN host isolation manager I0326 23:32:36.781207 3878 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0326 23:32:36.816686 3878 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0326 23:32:36.816783 3878 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0326 23:32:36.816804 3878 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0326 23:32:36.851263 3878 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0326 23:32:36.851328 3878 default_node_network_controller.go:808] Node ip-10-0-2-151.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0326 23:32:36.856736 3878 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.151"] k8s.ovn.org/zone-name:ip-10-0-2-151.ec2.internal] on node ip-10-0-2-151.ec2.internal I0326 23:32:36.876315 3878 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0326 23:32:36.881673 3878 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0326 23:32:36.881827 3878 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0326 23:32:36.881966 3878 gateway_init.go:263] Preparing Gateway I0326 23:32:36.881978 3878 gateway_shared_intf.go:1658] Creating new gateway I0326 23:32:36.941807 3878 controller_manager.go:427] Waiting for node in zone sync took: 250.752ms I0326 23:32:36.946855 3878 controller_manager.go:608] Switching to transit router for layer2 networks I0326 23:32:36.946870 3878 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-151.ec2.internal I0326 23:32:36.960171 3878 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:32:36.960278 3878 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:32:36.966603 3878 controller_manager.go:288] SCTP support detected in OVN I0326 23:32:36.967175 3878 services_controller.go:69] Creating services controller for network=default I0326 23:32:36.967235 3878 udn_enabled_svc.go:80] Waiting for service informer to sync I0326 23:32:36.967319 3878 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0326 23:32:36.967372 3878 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0326 23:32:36.967408 3878 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0326 23:32:36.967433 3878 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:32:36.967514 3878 iptables.go:108] Creating table: filter chain: OUTPUT I0326 23:32:36.967611 3878 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0326 23:32:36.967704 3878 udn_enabled_svc.go:95] Performing full resync I0326 23:32:36.967725 3878 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 63.424µs I0326 23:32:36.967760 3878 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0326 23:32:36.967806 3878 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 35.895µs I0326 23:32:36.967820 3878 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 60.488µs I0326 23:32:36.967827 3878 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0326 23:32:36.967839 3878 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0326 23:32:36.967848 3878 default_network_controller.go:321] Starting the default network controller I0326 23:32:36.967865 3878 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 13.404µs I0326 23:32:36.968114 3878 udn_enabled_svc.go:99] Waiting for handler to sync I0326 23:32:36.968132 3878 udn_enabled_svc.go:104] Starting worker I0326 23:32:36.975005 3878 gateway_shared_intf.go:1787] Gateway Creation Complete I0326 23:32:36.975020 3878 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-151.ec2.internal","mac-address":"0e:d0:e1:40:b8:33","ip-addresses":["10.0.2.151/24"],"ip-address":"10.0.2.151/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:052a1953-717c-4660-9497-79ea437c0357 k8s.ovn.org/node-encap-ips:["10.0.2.151"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.151/24"} k8s.ovn.org/zone-name:ip-10-0-2-151.ec2.internal] on node ip-10-0-2-151.ec2.internal I0326 23:32:36.977998 3878 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0326 23:32:36.978058 3878 repair.go:33] Syncing exgw routes took 43.287µs I0326 23:32:36.978078 3878 default_network_controller.go:407] Starting all the Watchers... I0326 23:32:36.978227 3878 namespace.go:98] [openshift-config] adding namespace I0326 23:32:36.978239 3878 namespace.go:98] [openshift-config-managed] adding namespace I0326 23:32:36.978239 3878 namespace.go:98] [openshift-backplane-cee] adding namespace I0326 23:32:36.978248 3878 namespace.go:98] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] adding namespace I0326 23:32:36.978250 3878 namespace.go:98] [openshift-backplane-cse] adding namespace I0326 23:32:36.978268 3878 namespace.go:98] [openshift-network-node-identity] adding namespace I0326 23:32:36.978266 3878 namespace.go:98] [openshift-kube-apiserver] adding namespace I0326 23:32:36.978274 3878 namespace.go:98] [openshift-logging] adding namespace I0326 23:32:36.978284 3878 namespace.go:98] [openshift-machine-api] adding namespace I0326 23:32:36.978284 3878 namespace.go:98] [dedicated-admin] adding namespace I0326 23:32:36.978289 3878 namespace.go:98] [openshift-dns] adding namespace I0326 23:32:36.978290 3878 namespace.go:98] [kube-system] adding namespace I0326 23:32:36.978293 3878 namespace.go:98] [openshift-authentication] adding namespace I0326 23:32:36.978302 3878 namespace.go:98] [openshift-kube-scheduler] adding namespace I0326 23:32:36.978235 3878 namespace.go:98] [openshift] adding namespace I0326 23:32:36.978905 3878 namespace.go:102] [openshift-config] adding namespace took 661.073µs I0326 23:32:36.978923 3878 namespace.go:98] [openshift-infra] adding namespace I0326 23:32:36.979367 3878 namespace.go:102] [openshift-infra] adding namespace took 437.175µs I0326 23:32:36.979383 3878 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0326 23:32:36.980035 3878 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 644.692µs I0326 23:32:36.980056 3878 namespace.go:98] [openshift-package-operator] adding namespace I0326 23:32:36.980635 3878 namespace.go:102] [openshift-config-managed] adding namespace took 2.384551ms I0326 23:32:36.980652 3878 namespace.go:98] [openshift-backplane-tam] adding namespace I0326 23:32:36.981161 3878 namespace.go:102] [openshift-backplane-cee] adding namespace took 2.907922ms I0326 23:32:36.981177 3878 namespace.go:98] [openshift-route-controller-manager] adding namespace I0326 23:32:36.981702 3878 namespace.go:102] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] adding namespace took 3.442879ms I0326 23:32:36.981721 3878 namespace.go:98] [openshift-ingress-operator] adding namespace I0326 23:32:36.982268 3878 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.003963ms I0326 23:32:36.982284 3878 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0326 23:32:36.982770 3878 namespace.go:102] [openshift-network-node-identity] adding namespace took 4.496037ms I0326 23:32:36.982785 3878 namespace.go:98] [openshift-console] adding namespace I0326 23:32:36.983286 3878 namespace.go:102] [openshift-kube-apiserver] adding namespace took 5.005306ms I0326 23:32:36.983305 3878 namespace.go:98] [default] adding namespace I0326 23:32:36.983889 3878 namespace.go:102] [openshift-logging] adding namespace took 5.605282ms I0326 23:32:36.983904 3878 namespace.go:98] [openshift-etcd] adding namespace I0326 23:32:36.984907 3878 namespace.go:102] [openshift-machine-api] adding namespace took 6.613736ms I0326 23:32:36.984928 3878 namespace.go:98] [openshift-service-ca-operator] adding namespace I0326 23:32:36.985412 3878 namespace.go:102] [dedicated-admin] adding namespace took 7.095158ms I0326 23:32:36.985431 3878 namespace.go:98] [openshift-console-operator] adding namespace I0326 23:32:36.986011 3878 namespace.go:102] [openshift-dns] adding namespace took 7.714611ms I0326 23:32:36.986027 3878 namespace.go:98] [openshift-ingress] adding namespace I0326 23:32:36.986625 3878 namespace.go:102] [kube-system] adding namespace took 8.324842ms I0326 23:32:36.986643 3878 namespace.go:98] [openshift-authentication-operator] adding namespace I0326 23:32:36.987073 3878 namespace.go:102] [openshift-authentication] adding namespace took 8.769876ms I0326 23:32:36.987084 3878 namespace.go:98] [openshift-network-operator] adding namespace I0326 23:32:36.987565 3878 namespace.go:102] [openshift-kube-scheduler] adding namespace took 9.256094ms I0326 23:32:36.987580 3878 namespace.go:98] [openshift-monitoring] adding namespace I0326 23:32:36.988049 3878 namespace.go:102] [openshift] adding namespace took 9.707595ms I0326 23:32:36.988066 3878 namespace.go:98] [openshift-backplane-mobb] adding namespace I0326 23:32:36.988523 3878 namespace.go:102] [openshift-package-operator] adding namespace took 8.45922ms I0326 23:32:36.988539 3878 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0326 23:32:36.988915 3878 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.258818ms I0326 23:32:36.988928 3878 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0326 23:32:36.989405 3878 namespace.go:102] [openshift-route-controller-manager] adding namespace took 8.207565ms I0326 23:32:36.989424 3878 namespace.go:98] [openshift-operators-redhat] adding namespace I0326 23:32:36.989922 3878 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.195423ms I0326 23:32:36.989938 3878 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0326 23:32:36.990510 3878 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.219882ms I0326 23:32:36.990525 3878 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0326 23:32:36.990977 3878 namespace.go:102] [openshift-console] adding namespace took 8.187ms I0326 23:32:36.990989 3878 namespace.go:98] [openshift-multus] adding namespace I0326 23:32:36.991472 3878 namespace.go:102] [default] adding namespace took 8.15877ms I0326 23:32:36.991488 3878 namespace.go:98] [kube-node-lease] adding namespace I0326 23:32:36.991959 3878 namespace.go:102] [openshift-etcd] adding namespace took 8.049418ms I0326 23:32:36.991970 3878 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0326 23:32:36.992595 3878 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.658282ms I0326 23:32:36.992614 3878 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0326 23:32:36.993328 3878 namespace.go:102] [openshift-console-operator] adding namespace took 7.886978ms I0326 23:32:36.993350 3878 namespace.go:98] [openshift-backplane] adding namespace I0326 23:32:36.993867 3878 namespace.go:102] [openshift-ingress] adding namespace took 7.833364ms I0326 23:32:36.993885 3878 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0326 23:32:36.994522 3878 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.873719ms I0326 23:32:36.994537 3878 namespace.go:98] [openshift-image-registry] adding namespace I0326 23:32:36.995177 3878 namespace.go:102] [openshift-network-operator] adding namespace took 8.085799ms I0326 23:32:36.995196 3878 namespace.go:98] [openshift-dns-operator] adding namespace I0326 23:32:36.995776 3878 namespace.go:102] [openshift-monitoring] adding namespace took 8.187776ms I0326 23:32:36.995779 3878 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:36.995797 3878 namespace.go:98] [openshift-backplane-srep] adding namespace I0326 23:32:36.996295 3878 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.221033ms I0326 23:32:36.996314 3878 namespace.go:98] [openshift-operators] adding namespace I0326 23:32:36.996851 3878 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.304165ms I0326 23:32:36.997528 3878 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 8.595167ms I0326 23:32:36.997543 3878 namespace.go:98] [openshift-node] adding namespace I0326 23:32:36.998017 3878 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.585846ms I0326 23:32:36.998031 3878 namespace.go:98] [kube-public] adding namespace I0326 23:32:36.998626 3878 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.682748ms I0326 23:32:36.998641 3878 namespace.go:98] [openshift-backplane-srep-ro] adding namespace W0326 23:32:36.999164 3878 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-4wfbf: pod openshift-multus/network-metrics-daemon-4wfbf: no pod IPs found I0326 23:32:36.999206 3878 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 8.672569ms I0326 23:32:36.999226 3878 namespace.go:98] [openshift-marketplace] adding namespace I0326 23:32:36.999802 3878 namespace.go:102] [openshift-multus] adding namespace took 8.807293ms I0326 23:32:36.999819 3878 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0326 23:32:37.000427 3878 namespace.go:102] [kube-node-lease] adding namespace took 8.932736ms I0326 23:32:37.000446 3878 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0326 23:32:37.000968 3878 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 8.98977ms I0326 23:32:37.000991 3878 namespace.go:98] [openshift-network-console] adding namespace I0326 23:32:37.001568 3878 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 8.94691ms I0326 23:32:37.001584 3878 namespace.go:98] [openshift-network-diagnostics] adding namespace I0326 23:32:37.002132 3878 namespace.go:102] [openshift-backplane] adding namespace took 8.776801ms I0326 23:32:37.002146 3878 namespace.go:98] [openshift-cluster-version] adding namespace I0326 23:32:37.002680 3878 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.788672ms I0326 23:32:37.002694 3878 namespace.go:98] [openshift-insights] adding namespace I0326 23:32:37.003716 3878 namespace.go:102] [openshift-image-registry] adding namespace took 9.172863ms I0326 23:32:37.003734 3878 namespace.go:98] [openshift-host-network] adding namespace I0326 23:32:37.004318 3878 namespace.go:102] [openshift-dns-operator] adding namespace took 9.117075ms I0326 23:32:37.004329 3878 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0326 23:32:37.004894 3878 namespace.go:102] [openshift-backplane-srep] adding namespace took 9.089906ms I0326 23:32:37.004907 3878 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0326 23:32:37.005420 3878 namespace.go:102] [openshift-operators] adding namespace took 9.098887ms I0326 23:32:37.005438 3878 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0326 23:32:37.005888 3878 namespace.go:102] [openshift-node] adding namespace took 8.337448ms I0326 23:32:37.005907 3878 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0326 23:32:37.006357 3878 namespace.go:102] [kube-public] adding namespace took 8.318504ms I0326 23:32:37.006372 3878 namespace.go:98] [openshift-controller-manager] adding namespace I0326 23:32:37.007001 3878 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.352562ms I0326 23:32:37.007020 3878 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0326 23:32:37.007637 3878 namespace.go:102] [openshift-marketplace] adding namespace took 8.401881ms I0326 23:32:37.007654 3878 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0326 23:32:37.008229 3878 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 8.402347ms I0326 23:32:37.008248 3878 namespace.go:98] [openshift-apiserver] adding namespace I0326 23:32:37.008763 3878 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.311608ms I0326 23:32:37.008778 3878 namespace.go:98] [openshift-machine-config-operator] adding namespace W0326 23:32:37.009312 3878 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-vshpw: pod openshift-network-diagnostics/network-check-target-vshpw: no pod IPs found I0326 23:32:37.009315 3878 namespace.go:102] [openshift-network-console] adding namespace took 8.31718ms I0326 23:32:37.009331 3878 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0326 23:32:37.009905 3878 namespace.go:102] [openshift-network-diagnostics] adding namespace took 8.31445ms I0326 23:32:37.009918 3878 namespace.go:98] [openshift-customer-monitoring] adding namespace I0326 23:32:37.010421 3878 namespace.go:102] [openshift-cluster-version] adding namespace took 8.266992ms I0326 23:32:37.010440 3878 namespace.go:98] [openshift-must-gather-operator] adding namespace I0326 23:32:37.010919 3878 namespace.go:102] [openshift-insights] adding namespace took 8.21722ms I0326 23:32:37.010937 3878 namespace.go:98] [openshift-backplane-csm] adding namespace I0326 23:32:37.011491 3878 namespace.go:102] [openshift-host-network] adding namespace took 7.750487ms I0326 23:32:37.011505 3878 namespace.go:98] [openshift-console-user-settings] adding namespace I0326 23:32:37.012107 3878 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.770019ms I0326 23:32:37.012131 3878 namespace.go:98] [openshift-ingress-canary] adding namespace I0326 23:32:37.012701 3878 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.787046ms I0326 23:32:37.012717 3878 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0326 23:32:37.013338 3878 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.892971ms I0326 23:32:37.013356 3878 namespace.go:98] [openshift-apiserver-operator] adding namespace I0326 23:32:37.013970 3878 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 8.056044ms I0326 23:32:37.014538 3878 namespace.go:102] [openshift-controller-manager] adding namespace took 8.158455ms I0326 23:32:37.015095 3878 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.067259ms I0326 23:32:37.015113 3878 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0326 23:32:37.015588 3878 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.927929ms I0326 23:32:37.016106 3878 namespace.go:102] [openshift-apiserver] adding namespace took 7.851145ms I0326 23:32:37.016614 3878 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.8294ms I0326 23:32:37.017103 3878 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.767776ms I0326 23:32:37.017565 3878 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.640737ms I0326 23:32:37.018017 3878 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.570087ms I0326 23:32:37.018510 3878 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.568157ms I0326 23:32:37.018523 3878 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0326 23:32:37.018985 3878 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.474639ms I0326 23:32:37.019434 3878 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.297991ms I0326 23:32:37.019944 3878 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.220487ms I0326 23:32:37.019960 3878 namespace.go:98] [openshift-config-operator] adding namespace I0326 23:32:37.020417 3878 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.056348ms I0326 23:32:37.020430 3878 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0326 23:32:37.020983 3878 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.864848ms I0326 23:32:37.021508 3878 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 2.980302ms I0326 23:32:37.022016 3878 namespace.go:102] [openshift-config-operator] adding namespace took 2.048959ms I0326 23:32:37.022480 3878 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 2.044774ms I0326 23:32:37.022888 3878 zone_ic_handler.go:186] Time taken to create transit switch: 306.07µs I0326 23:32:37.022984 3878 master.go:532] Adding or Updating local node "ip-10-0-2-151.ec2.internal" for network "default" I0326 23:32:37.023612 3878 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-231.ec2.internal for the network default I0326 23:32:37.024599 3878 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-231.ec2.internal" on network "default" took: 997.798µs W0326 23:32:37.032850 3878 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-4wfbf: pod openshift-multus/network-metrics-daemon-4wfbf: no pod IPs found W0326 23:32:37.032872 3878 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-vshpw: pod openshift-network-diagnostics/network-check-target-vshpw: no pod IPs found I0326 23:32:37.033970 3878 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-151.ec2.internal for the network default I0326 23:32:37.034936 3878 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0326 23:32:37.034974 3878 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0326 23:32:37.034991 3878 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0326 23:32:37.035022 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-151.ec2.internal in network "default" I0326 23:32:37.035069 3878 node_tracker.go:169] Node ip-10-0-2-151.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:37.035079 3878 services_controller.go:574] Full service sync requested for network=default I0326 23:32:37.035087 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-231.ec2.internal in network "default" I0326 23:32:37.035113 3878 node_tracker.go:169] Node ip-10-0-0-231.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:37.035123 3878 services_controller.go:574] Full service sync requested for network=default I0326 23:32:37.135146 3878 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0326 23:32:37.135180 3878 services_controller.go:208] Setting up event handlers for services for network=default I0326 23:32:37.135280 3878 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0326 23:32:37.135327 3878 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0326 23:32:37.135344 3878 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0326 23:32:37.235438 3878 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0326 23:32:37.235459 3878 repair.go:60] Starting repairing loop for services I0326 23:32:37.235566 3878 repair.go:131] Deleted 0 stale service LBs I0326 23:32:37.235592 3878 repair.go:137] Deleted 0 stale Chassis Template Vars I0326 23:32:37.235609 3878 repair.go:62] Finished repairing loop for services: 151.824µs I0326 23:32:37.235639 3878 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0326 23:32:37.235650 3878 services_controller.go:254] Starting workers for network=default I0326 23:32:37.235972 3878 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-4wfbf] creating logical port openshift-multus_network-metrics-daemon-4wfbf for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:37.236088 3878 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-4wfbf I0326 23:32:37.236356 3878 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-vshpw] creating logical port openshift-network-diagnostics_network-check-target-vshpw for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:37.236472 3878 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-vshpw I0326 23:32:37.251281 3878 pod.go:62] [openshift-network-diagnostics/network-check-target-vshpw] pod update took 14.842709ms I0326 23:32:37.251303 3878 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-vshpw] addLogicalPort annotation time took 14.868694ms I0326 23:32:37.251975 3878 pod.go:62] [openshift-multus/network-metrics-daemon-4wfbf] pod update took 15.925197ms I0326 23:32:37.251997 3878 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-4wfbf] addLogicalPort annotation time took 15.950815ms I0326 23:32:37.252423 3878 pods.go:271] [openshift-network-diagnostics/network-check-target-vshpw] addLogicalPort took 16.077078ms, libovsdb time 780.373µs I0326 23:32:37.252854 3878 pods.go:271] [openshift-multus/network-metrics-daemon-4wfbf] addLogicalPort took 16.892242ms, libovsdb time 634.053µs I0326 23:32:37.253278 3878 admin_network_policy_controller.go:227] Starting controller default-network-controller I0326 23:32:37.253302 3878 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0326 23:32:37.253314 3878 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0326 23:32:37.253318 3878 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0326 23:32:37.253545 3878 repair.go:29] Repairing admin network policies took 221.589µs I0326 23:32:37.253656 3878 repair.go:92] Repairing baseline admin network policies took 99.682µs I0326 23:32:37.254056 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0326 23:32:37.254080 3878 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0326 23:32:37.256134 3878 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0326 23:32:37.256159 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0326 23:32:37.256169 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0326 23:32:37.258656 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0326 23:32:37.258683 3878 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0326 23:32:37.258700 3878 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0326 23:32:37.261133 3878 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0326 23:32:37.261154 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0326 23:32:37.261164 3878 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0326 23:32:37.261686 3878 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0326 23:32:37.261708 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0326 23:32:37.262663 3878 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0326 23:32:37.262677 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:32:37.263603 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0326 23:32:37.263624 3878 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0326 23:32:37.263635 3878 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0326 23:32:37.265715 3878 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0326 23:32:37.265739 3878 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0326 23:32:37.265749 3878 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0326 23:32:37.266744 3878 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0326 23:32:37.266765 3878 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0326 23:32:37.266776 3878 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0326 23:32:37.268597 3878 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0326 23:32:37.268617 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0326 23:32:37.268627 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0326 23:32:37.270122 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0326 23:32:37.270140 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0326 23:32:37.270148 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0326 23:32:37.270624 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0326 23:32:37.270647 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0326 23:32:37.270671 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:32:37.271362 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0326 23:32:37.271382 3878 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0326 23:32:37.271408 3878 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0326 23:32:37.273515 3878 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0326 23:32:37.273532 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0326 23:32:37.273539 3878 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0326 23:32:37.273812 3878 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0326 23:32:37.273829 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0326 23:32:37.273836 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0326 23:32:37.274328 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:32:37.274351 3878 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0326 23:32:37.274361 3878 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0326 23:32:37.274868 3878 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0326 23:32:37.274886 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0326 23:32:37.274895 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0326 23:32:37.275380 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:32:37.275421 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0326 23:32:37.275431 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0326 23:32:37.275728 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0326 23:32:37.275743 3878 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0326 23:32:37.275756 3878 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0326 23:32:37.276731 3878 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0326 23:32:37.276749 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0326 23:32:37.276756 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0326 23:32:37.277277 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0326 23:32:37.277294 3878 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0326 23:32:37.277302 3878 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0326 23:32:37.278291 3878 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0326 23:32:37.278306 3878 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0326 23:32:37.278315 3878 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0326 23:32:37.279179 3878 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0326 23:32:37.279197 3878 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0326 23:32:37.279209 3878 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0326 23:32:37.280615 3878 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0326 23:32:37.280639 3878 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0326 23:32:37.280647 3878 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0326 23:32:37.280981 3878 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0326 23:32:37.280998 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0326 23:32:37.281005 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0326 23:32:37.281503 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0326 23:32:37.281520 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0326 23:32:37.281532 3878 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0326 23:32:37.282173 3878 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0326 23:32:37.282189 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0326 23:32:37.282196 3878 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0326 23:32:37.282669 3878 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0326 23:32:37.282685 3878 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0326 23:32:37.282691 3878 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0326 23:32:37.283950 3878 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0326 23:32:37.283967 3878 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0326 23:32:37.283974 3878 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0326 23:32:37.284231 3878 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0326 23:32:37.284245 3878 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0326 23:32:37.284257 3878 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0326 23:32:37.285997 3878 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0326 23:32:37.291503 3878 egressfirewall.go:319] Starting EgressFirewall controller I0326 23:32:37.291534 3878 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0326 23:32:37.291562 3878 controller.go:133] Adding controller egress-firewall-controller event handlers I0326 23:32:37.291573 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0326 23:32:37.291591 3878 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0326 23:32:37.291591 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0326 23:32:37.291601 3878 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0326 23:32:37.291895 3878 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0326 23:32:37.291914 3878 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0326 23:32:37.291925 3878 egressqos.go:193] Setting up event handlers for EgressQoS I0326 23:32:37.292000 3878 egressfirewall.go:628] Syncing node "ip-10-0-0-231.ec2.internal" for egress firewall I0326 23:32:37.292013 3878 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:37.292042 3878 egressqos.go:245] Starting EgressQoS Controller I0326 23:32:37.292057 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0326 23:32:37.292069 3878 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0326 23:32:37.292074 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0326 23:32:37.292079 3878 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0326 23:32:37.292084 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0326 23:32:37.292088 3878 shared_informer.go:356] "Caches are synced" controller="egressqos" I0326 23:32:37.292092 3878 egressqos.go:259] Repairing EgressQoSes I0326 23:32:37.292096 3878 egressqos.go:399] Starting repairing loop for egressqos I0326 23:32:37.292177 3878 egressqos.go:401] Finished repairing loop for egressqos: 80.929µs I0326 23:32:37.292197 3878 egressservice_zone.go:132] Setting up event handlers for Egress Services I0326 23:32:37.292233 3878 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-151.ec2.internal I0326 23:32:37.292285 3878 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-151.ec2.internal : 52.362µs I0326 23:32:37.292339 3878 egressservice_zone.go:210] Starting Egress Services Controller I0326 23:32:37.292354 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:32:37.292362 3878 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:32:37.292368 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:32:37.292373 3878 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:32:37.292379 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:32:37.292384 3878 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:32:37.292404 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0326 23:32:37.292410 3878 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0326 23:32:37.292415 3878 egressservice_zone.go:228] Repairing Egress Services I0326 23:32:37.292681 3878 master_controller.go:88] Starting Admin Policy Based Route Controller I0326 23:32:37.292690 3878 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:32:37.292753 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:37.292764 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 12.922µs I0326 23:32:37.292771 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-231.ec2.internal I0326 23:32:37.292774 3878 default_network_controller.go:568] Completing all the Watchers took 314.688442ms I0326 23:32:37.292780 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-231.ec2.internal: 9.821µs I0326 23:32:37.292783 3878 default_network_controller.go:572] Starting unidling controllers I0326 23:32:37.292795 3878 unidle.go:46] Registering OVN SB ControllerEvent handler I0326 23:32:37.292802 3878 unidle.go:63] Populating Initial ContollerEvent events I0326 23:32:37.292811 3878 unidle.go:79] Setting up event handlers for services I0326 23:32:37.292861 3878 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:32:37.496817 3878 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:37.996431 3878 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:38.496463 3878 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:38.996070 3878 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:39.363559 3878 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:32:39.496805 3878 default_node_network_controller.go:518] Node connection status = connected I0326 23:32:39.513306 3878 bridgeconfig.go:537] Gateway is ready I0326 23:32:39.513326 3878 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0326 23:32:39.524128 3878 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.151. Updating OVN encap IP. I0326 23:32:39.528999 3878 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.151 - it is already configured I0326 23:32:39.529015 3878 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.151/24:{} 172.20.0.1/32:{}]. Updating annotations. I0326 23:32:39.529322 3878 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.151/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-151.ec2.internal","mac-address":"0e:d0:e1:40:b8:33","ip-addresses":["10.0.2.151/24"],"ip-address":"10.0.2.151/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:052a1953-717c-4660-9497-79ea437c0357 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.151/24"}] on node ip-10-0-2-151.ec2.internal I0326 23:32:39.544647 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-151.ec2.internal in network "default" I0326 23:32:39.544670 3878 master.go:532] Adding or Updating local node "ip-10-0-2-151.ec2.internal" for network "default" I0326 23:32:39.544684 3878 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:39.544719 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:39.544722 3878 node_tracker.go:169] Node ip-10-0-2-151.ec2.internal switch + router changed, syncing services in network "default" I0326 23:32:39.544735 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 20.151µs I0326 23:32:39.544747 3878 services_controller.go:574] Full service sync requested for network=default I0326 23:32:39.545041 3878 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.151 I0326 23:32:39.545144 3878 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-151.ec2.internal I0326 23:32:39.550172 3878 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-151.ec2.internal. Updating no re-route policies I0326 23:32:39.550241 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-k5xq9 I0326 23:32:39.550262 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-k5xq9 I0326 23:32:39.550269 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-zfq6b I0326 23:32:39.550268 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-tdhx2 I0326 23:32:39.550283 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-949z9 I0326 23:32:39.550293 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-tdhx2 I0326 23:32:39.550295 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-k5xq9 after 0 failed attempt(s) I0326 23:32:39.550297 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-4wfbf I0326 23:32:39.550311 3878 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-br8q9 I0326 23:32:39.550314 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-4wfbf I0326 23:32:39.550319 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-tdhx2 after 0 failed attempt(s) I0326 23:32:39.550320 3878 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-br8q9 I0326 23:32:39.550320 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-h86wb I0326 23:32:39.550312 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-151.ec2.internal I0326 23:32:39.550335 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-h86wb I0326 23:32:39.550339 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-nqpt8 I0326 23:32:39.550341 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-151.ec2.internal I0326 23:32:39.550341 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-8wzlm I0326 23:32:39.550350 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-nqpt8 I0326 23:32:39.550354 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-8wzlm I0326 23:32:39.550353 3878 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-4rxqc I0326 23:32:39.550361 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-151.ec2.internal after 0 failed attempt(s) I0326 23:32:39.550369 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-nqpt8 after 0 failed attempt(s) I0326 23:32:39.550352 3878 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-4wfbf] creating logical port openshift-multus_network-metrics-daemon-4wfbf for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:39.550417 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-vshpw I0326 23:32:39.550433 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-vshpw I0326 23:32:39.550436 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-zfq6b I0326 23:32:39.550463 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-zfq6b after 0 failed attempt(s) I0326 23:32:39.550371 3878 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-4rxqc I0326 23:32:39.550488 3878 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-4rxqc after 0 failed attempt(s) I0326 23:32:39.550299 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-949z9 I0326 23:32:39.550507 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-949z9 after 0 failed attempt(s) I0326 23:32:39.550512 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-qzhhf I0326 23:32:39.550533 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-qzhhf I0326 23:32:39.550350 3878 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-br8q9 after 0 failed attempt(s) I0326 23:32:39.550539 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-h86wb after 0 failed attempt(s) I0326 23:32:39.550555 3878 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-151.ec2.internal I0326 23:32:39.550557 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-8wzlm after 0 failed attempt(s) I0326 23:32:39.550568 3878 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-151.ec2.internal I0326 23:32:39.550573 3878 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-vshpw] creating logical port openshift-network-diagnostics_network-check-target-vshpw for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:39.550576 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-qzhhf after 0 failed attempt(s) I0326 23:32:39.550607 3878 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-151.ec2.internal after 0 failed attempt(s) I0326 23:32:39.551066 3878 pods.go:271] [openshift-multus/network-metrics-daemon-4wfbf] addLogicalPort took 722.058µs, libovsdb time 244.058µs I0326 23:32:39.551078 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-4wfbf after 0 failed attempt(s) I0326 23:32:39.551284 3878 pods.go:271] [openshift-network-diagnostics/network-check-target-vshpw] addLogicalPort took 718.309µs, libovsdb time 263.997µs I0326 23:32:39.551297 3878 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-vshpw after 0 failed attempt(s) I0326 23:32:39.562920 3878 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0326 23:32:39.575422 3878 iptables.go:108] Creating table: mangle chain: OUTPUT I0326 23:32:39.579078 3878 iptables.go:108] Creating table: nat chain: OUTPUT I0326 23:32:39.604560 3878 iptables.go:108] Creating table: nat chain: PREROUTING I0326 23:32:39.646804 3878 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:32:39.665267 3878 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0326 23:32:39.665445 3878 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0326 23:32:39.665499 3878 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0326 23:32:39.665570 3878 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0326 23:32:39.665622 3878 gateway_init.go:325] Gateway and management port readiness took 2.673406734s I0326 23:32:39.665653 3878 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0326 23:32:39.665694 3878 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0326 23:32:39.665710 3878 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0326 23:32:39.665721 3878 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:32:39.665784 3878 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0326 23:32:39.665861 3878 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 49.301µs I0326 23:32:39.666127 3878 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0326 23:32:39.666151 3878 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.046µs I0326 23:32:39.666159 3878 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 32.164µs I0326 23:32:39.666164 3878 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0326 23:32:39.666175 3878 nad_controller.go:173] [node-nad-controller NAD controller]: started I0326 23:32:39.666181 3878 default_node_network_controller.go:898] Starting the default node network controller I0326 23:32:39.666187 3878 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0326 23:32:39.666279 3878 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 18.741µs I0326 23:32:39.666348 3878 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0326 23:32:39.666356 3878 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0326 23:32:39.666384 3878 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c047e0 0xc0002dc4d0 0xc0002dc540 0xc0002dc5b0 0xc0002dc620 0xc0002dc690 0xc0002dc700 0xc000c14b90 0xc0002dc770 0xc0002dc7e0 0xc000114320 0xc0061d9180 0xc0002dc850 map[0x2f67040:0xc002814f60 0x2f67580:0xc0028150e0 0x2f68000:0xc0007a55c0 0x2f682a0:0xc002814f00 0x2f687e0:0xc002814fc0 0x2f68a80:0xc002815020 0x2f68d20:0xc002815080 0x2f69500:0xc002815140 0x2f697a0:0xc0028151a0 0x2f93d40:0xc0007a5320 0x2f940e0:0xc0007a5380 0x2f94bc0:0xc000c6c000 0x2f96520:0xc001b62000 0x2f97000:0xc0007a5560 0x2f97e80:0xc000b0c9c0] 0xc000432a10 0} I0326 23:32:39.666468 3878 gateway.go:151] Starting gateway service sync I0326 23:32:39.666920 3878 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0326 23:32:39.669415 3878 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0326 23:32:39.679640 3878 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0326 23:32:39.680884 3878 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0326 23:32:39.682140 3878 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0326 23:32:39.716744 3878 gateway.go:168] Gateway service sync done. Time taken: 50.262273ms I0326 23:32:39.717226 3878 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32538 I0326 23:32:39.717655 3878 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c047e0 0xc0002dc4d0 0xc0002dc540 0xc0002dc5b0 0xc0002dc620 0xc0002dc690 0xc0002dc700 0xc000c14b90 0xc0002dc770 0xc0002dc7e0 0xc000114320 0xc0061d9180 0xc0002dc850 map[0x2f67040:0xc002814f60 0x2f67580:0xc0028150e0 0x2f68000:0xc0007a55c0 0x2f682a0:0xc002814f00 0x2f687e0:0xc002814fc0 0x2f68a80:0xc002815020 0x2f68d20:0xc002815080 0x2f69500:0xc002815140 0x2f697a0:0xc0028151a0 0x2f93d40:0xc0007a5320 0x2f940e0:0xc0007a5380 0x2f94bc0:0xc000c6c000 0x2f96520:0xc001b62000 0x2f97000:0xc0007a5560 0x2f97e80:0xc000b0c9c0] 0xc000432a10 0} I0326 23:32:39.718172 3878 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c047e0 0xc0002dc4d0 0xc0002dc540 0xc0002dc5b0 0xc0002dc620 0xc0002dc690 0xc0002dc700 0xc000c14b90 0xc0002dc770 0xc0002dc7e0 0xc000114320 0xc0061d9180 0xc0002dc850 map[0x2f67040:0xc002814f60 0x2f67580:0xc0028150e0 0x2f68000:0xc0007a55c0 0x2f682a0:0xc002814f00 0x2f687e0:0xc002814fc0 0x2f68a80:0xc002815020 0x2f68d20:0xc002815080 0x2f69500:0xc002815140 0x2f697a0:0xc0028151a0 0x2f93d40:0xc0007a5320 0x2f940e0:0xc0007a5380 0x2f94bc0:0xc000c6c000 0x2f96520:0xc001b62000 0x2f97000:0xc0007a5560 0x2f97e80:0xc000b0c9c0] 0xc000432a10 0} I0326 23:32:39.718215 3878 gateway.go:519] Reconciling gateway with updates I0326 23:32:39.723499 3878 gateway.go:326] Spawning Conntrack Rule Check Thread I0326 23:32:39.723585 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:39.723593 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:39.723606 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:39.723610 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:39.723614 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:39.723623 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:39.723623 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:32:39.723648 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:32:39.723659 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:32:39.723655 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:39.723670 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:39.723674 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:32:39.723677 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:39.723679 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:39.723678 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:39.723684 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:32:39.723687 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:39.723697 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:39.723699 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:32:39.723702 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:39.723703 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:39.723707 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:32:39.723711 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:39.723695 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:39.723712 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:39.723722 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:39.723725 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:39.723732 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:39.723733 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:39.723737 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:39.723741 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:32:39.723744 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:32:39.723738 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:39.723747 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:39.723747 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:39.723759 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:39.723757 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:39.723764 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:39.723764 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:39.723767 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:39.723723 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:39.723776 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:32:39.723773 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:39.723783 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:32:39.723785 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:39.723787 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:32:39.723701 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:39.723795 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:32:39.723797 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:39.723773 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:39.723803 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:39.723828 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:39.723842 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:32:39.723775 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:39.723856 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:39.723862 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:39.723866 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:32:39.723872 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:39.723877 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:32:39.723722 3878 port_claim.go:71] Svc openshift-ingress/router-default: port 32508 is already open, no action needed I0326 23:32:39.723893 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:32:39.723697 3878 default_node_network_controller.go:943] Gateway and management port readiness took 57.269951ms I0326 23:32:39.723908 3878 port_claim.go:71] Svc openshift-ingress/router-default: port 30435 is already open, no action needed I0326 23:32:39.723761 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:32:39.723921 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:32:39.723736 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:39.723940 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:39.723779 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:39.723724 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:32:39.723800 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:39.723776 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:39.724100 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:39.724109 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:39.724112 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:39.724099 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:39.724119 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:39.724123 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:39.724128 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:39.724128 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:32:39.724129 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:39.724137 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:39.724141 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:32:39.724120 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:39.724145 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:39.724147 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:39.724114 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:39.724157 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:39.724157 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:39.724162 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:39.724166 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:32:39.724170 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:39.724173 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:39.724136 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:39.724178 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:39.724154 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:39.724187 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:39.724186 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:39.724186 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:39.724201 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:32:39.724203 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:39.724207 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:39.724212 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:39.724215 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:32:39.724213 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:39.724224 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:39.724227 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:32:39.724231 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:39.724234 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:32:39.724215 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:39.724142 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:39.724321 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:39.724299 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:39.725064 3878 node_ip_handler_linux.go:160] Node IP manager is running W0326 23:32:39.769883 3878 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:32:39.770276 3878 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:32:39.770643 3878 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0326 23:32:39.779344 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:32:39.779426 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:32:39.779452 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:32:39.779473 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:32:39.779480 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:32:39.779488 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:32:39.779496 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:32:39.779501 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:32:39.779507 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:32:39.779512 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:32:39.779522 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:32:39.779535 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:32:39.779559 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:32:39.825840 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:32:39.825876 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:32:39.873931 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:32:39.873965 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:32:39.873981 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:32:39.873990 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:32:39.873996 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:32:39.874002 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:32:39.874007 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:32:39.874013 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) W0326 23:32:40.980484 3878 egressip_healthcheck.go:76] Health checking using insecure connection I0326 23:32:40.980529 3878 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0326 23:32:40.980490 3878 default_node_network_controller.go:1364] Starting node controller node sync I0326 23:32:40.992960 3878 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 12.362835ms I0326 23:32:40.993138 3878 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-231.ec2.internal", IP: 10.0.0.231 to PMTUD blocking rules I0326 23:32:40.993154 3878 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-231.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:32:40.998952 3878 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0326 23:32:40.999284 3878 egressservice_node.go:93] Setting up event handlers for Egress Services I0326 23:32:40.999440 3878 egressservice_node.go:183] Starting Egress Services Controller I0326 23:32:40.999456 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:32:40.999469 3878 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:32:40.999476 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:32:40.999481 3878 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:32:40.999487 3878 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:32:40.999510 3878 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:32:40.999520 3878 egressservice_node.go:197] Repairing Egress Services I0326 23:32:41.009657 3878 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0326 23:32:41.009668 3878 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:32:41.009918 3878 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0326 23:32:41.009930 3878 link_network_manager.go:117] Link manager is running I0326 23:32:41.009941 3878 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0326 23:32:41.010016 3878 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0326 23:32:41.010808 3878 vrf_manager.go:128] VRF manager is running I0326 23:32:41.011159 3878 node_controller_manager.go:437] Removing flows to drop GARP I0326 23:32:41.011169 3878 gateway.go:519] Reconciling gateway with updates I0326 23:32:41.035949 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:41.035965 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:41.035974 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:32:41.035977 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:41.035984 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:32:41.035995 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:32:41.035996 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:32:41.035962 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:41.036016 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:32:41.036015 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:41.036022 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:32:41.035999 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:41.036028 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:32:41.036034 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:32:41.036035 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:41.036042 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:32:41.036051 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:32:41.036055 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:41.036057 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:41.036064 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:41.036073 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:41.036075 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:32:41.036083 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:32:41.036093 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:41.036120 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:41.036132 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:41.036064 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:32:41.036137 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:32:41.036132 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:32:41.036099 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:32:41.036135 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:32:41.036150 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:41.036150 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:41.036159 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:32:41.036161 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:41.036165 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:32:41.036166 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:41.036170 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:32:41.036179 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:32:41.036179 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:32:41.036185 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:32:41.036103 3878 port_claim.go:71] Svc openshift-ingress/router-default: port 32508 is already open, no action needed I0326 23:32:41.036195 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:41.036172 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:32:41.036203 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:32:41.036205 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:32:41.036208 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:41.036215 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:32:41.036216 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:41.036218 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:41.036223 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:41.036227 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:32:41.036228 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:32:41.036229 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:41.036236 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:32:41.036067 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:32:41.036139 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:32:41.036244 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:32:41.036256 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:32:41.036256 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:41.036267 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:41.036267 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:41.036218 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:32:41.036272 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:41.036278 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:41.036284 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:32:41.036287 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:32:41.036292 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:41.036294 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:41.036300 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:32:41.036299 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:41.036205 3878 port_claim.go:71] Svc openshift-ingress/router-default: port 30435 is already open, no action needed I0326 23:32:41.036280 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:32:41.036075 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:32:41.036319 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:41.036321 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:41.036282 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:32:41.036323 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:32:41.036328 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:32:41.036335 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:32:41.036339 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:32:41.036156 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:32:41.036349 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:32:41.036119 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:41.036268 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:32:41.036362 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:32:41.036134 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:41.036374 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:32:41.036377 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:32:41.036379 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:32:41.036112 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:41.036407 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:32:41.036243 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:32:41.036142 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:32:41.036230 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:32:41.036271 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:32:41.036272 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:41.036278 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:32:41.036308 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:32:41.036506 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:32:41.036315 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:32:41.036522 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:32:41.036303 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:32:41.036319 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:41.036583 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:32:41.036022 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:41.036633 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:32:41.036377 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:32:41.036466 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:32:41.036322 3878 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:41.036671 3878 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:32:41.077851 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:32:41.077884 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:32:41.077893 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:32:41.077900 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:32:41.077905 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:32:41.077912 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:32:41.077917 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:32:41.077923 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:32:41.117136 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:32:41.117160 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:32:41.117186 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:32:41.117193 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:32:41.117200 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:32:41.117210 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:32:41.117216 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:32:41.117223 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:32:41.117239 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:32:41.117259 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:32:41.117268 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:32:41.117273 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:32:41.117278 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:32:41.117285 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:32:41.117292 3878 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:32:42.480732 3878 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:42.480802 3878 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:43.132139 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:43.132176 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 46.507µs I0326 23:32:43.156418 3878 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-566676bfc5-jlf6d] creating logical port openshift-ingress_router-default-566676bfc5-jlf6d for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:43.156567 3878 kube.go:256] Updating pod openshift-ingress/router-default-566676bfc5-jlf6d I0326 23:32:43.159832 3878 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5fd5865cf4-sbv9c] creating logical port openshift-image-registry_image-registry-5fd5865cf4-sbv9c for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:43.159975 3878 kube.go:256] Updating pod openshift-image-registry/image-registry-5fd5865cf4-sbv9c I0326 23:32:43.179881 3878 pod.go:62] [openshift-ingress/router-default-566676bfc5-jlf6d] pod update took 23.337721ms I0326 23:32:43.179907 3878 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-566676bfc5-jlf6d] addLogicalPort annotation time took 23.376386ms I0326 23:32:43.181107 3878 pods.go:271] [openshift-ingress/router-default-566676bfc5-jlf6d] addLogicalPort took 24.71874ms, libovsdb time 956.348µs I0326 23:32:43.184959 3878 pod.go:62] [openshift-image-registry/image-registry-5fd5865cf4-sbv9c] pod update took 25.01822ms I0326 23:32:43.184981 3878 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-5fd5865cf4-sbv9c] addLogicalPort annotation time took 25.059251ms I0326 23:32:43.186241 3878 pods.go:271] [openshift-image-registry/image-registry-5fd5865cf4-sbv9c] addLogicalPort took 26.420854ms, libovsdb time 1.051991ms I0326 23:32:43.187008 3878 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-z27r2] creating logical port openshift-ingress-canary_ingress-canary-z27r2 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:43.187143 3878 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-gcmpr] creating logical port openshift-dns_dns-default-gcmpr for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:43.187166 3878 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-z27r2 I0326 23:32:43.187210 3878 kube.go:256] Updating pod openshift-dns/dns-default-gcmpr I0326 23:32:43.204980 3878 pod.go:62] [openshift-ingress-canary/ingress-canary-z27r2] pod update took 17.836601ms I0326 23:32:43.205030 3878 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-z27r2] addLogicalPort annotation time took 17.860925ms I0326 23:32:43.205213 3878 pod.go:62] [openshift-dns/dns-default-gcmpr] pod update took 18.022532ms I0326 23:32:43.205230 3878 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-gcmpr] addLogicalPort annotation time took 18.042338ms I0326 23:32:43.206250 3878 pods.go:271] [openshift-ingress-canary/ingress-canary-z27r2] addLogicalPort took 19.253286ms, libovsdb time 940.941µs I0326 23:32:43.206612 3878 pods.go:271] [openshift-dns/dns-default-gcmpr] addLogicalPort took 19.480288ms, libovsdb time 932.436µs I0326 23:32:43.538450 3878 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:43.538479 3878 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-4rxqc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:43.549215 3878 cni.go:403] [openshift-ingress-canary/ingress-canary-z27r2 cb1cd93941764f1a04f6c37374a38657db40befd8e6ef86011a8b0c12e11aa5f network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-z27r2 cb1cd93941764f1a04f6c37374a38657db40befd8e6ef86011a8b0c12e11aa5f network default NAD default] I0326 23:32:43.584490 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-z27r2, hostIfaceName: cb1cd93941764f1, network: default, NAD default, SandboxID: "cb1cd93941764f1a04f6c37374a38657db40befd8e6ef86011a8b0c12e11aa5f", PCI device ID: , UID: "ecb981e0-b392-4807-8204-543b3d057786", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0326 23:32:43.626569 3878 cni.go:424] [openshift-ingress-canary/ingress-canary-z27r2 cb1cd93941764f1a04f6c37374a38657db40befd8e6ef86011a8b0c12e11aa5f network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-z27r2 cb1cd93941764f1a04f6c37374a38657db40befd8e6ef86011a8b0c12e11aa5f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:6e:27:48:bd:4e\",\"name\":\"cb1cd93941764f1\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bf444580-5548-4417-92fd-375edcb73104\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:32:45.539334 3878 cni.go:403] [openshift-network-diagnostics/network-check-target-vshpw 30bfb64f50506a29e18995233c3d54f6d8f5ec01ac7ab90c2e498fad34774b0b network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-vshpw 30bfb64f50506a29e18995233c3d54f6d8f5ec01ac7ab90c2e498fad34774b0b network default NAD default] I0326 23:32:45.548737 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-vshpw, hostIfaceName: 30bfb64f50506a2, network: default, NAD default, SandboxID: "30bfb64f50506a29e18995233c3d54f6d8f5ec01ac7ab90c2e498fad34774b0b", PCI device ID: , UID: "b934c75e-963f-45b4-a741-f3a6d93e462c", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0326 23:32:45.590563 3878 cni.go:424] [openshift-network-diagnostics/network-check-target-vshpw 30bfb64f50506a29e18995233c3d54f6d8f5ec01ac7ab90c2e498fad34774b0b network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-vshpw 30bfb64f50506a29e18995233c3d54f6d8f5ec01ac7ab90c2e498fad34774b0b network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:ab:0e:56:14:32\",\"name\":\"30bfb64f50506a2\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/02fc815d-04cc-41e7-be87-3705cd844b5a\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:32:47.078651 3878 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:32:47.126263 3878 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0326 23:32:49.499692 3878 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:32:49.853007 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:32:49.853030 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 30.214µs I0326 23:32:49.853072 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:32:49.853090 3878 node_tracker.go:220] Node ip-10-0-1-147.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0326 23:32:49.853090 3878 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall E0326 23:32:49.853158 3878 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0326 23:32:49.853178 3878 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-147.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-147.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-147.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-147.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-147.ec2.internal, error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0326 23:32:49.867414 3878 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input I0326 23:32:49.867446 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0326 23:32:49.867479 3878 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0326 23:32:49.879042 3878 egressfirewall.go:628] Syncing node "ip-10-0-2-151.ec2.internal" for egress firewall I0326 23:32:49.879064 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-151.ec2.internal I0326 23:32:49.879093 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-151.ec2.internal: 33.902µs I0326 23:32:49.881006 3878 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:32:49.881015 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:32:49.881026 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0326 23:32:49.881071 3878 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0326 23:32:49.881005 3878 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input I0326 23:32:49.881028 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 17.289µs E0326 23:32:49.959832 3878 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input E0326 23:32:49.959862 3878 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-147.ec2.internal annotation: unexpected end of JSON input I0326 23:32:49.959881 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0326 23:32:49.959939 3878 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-147.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-147.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-147.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-147.ec2.internal" I0326 23:32:49.970370 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0326 23:32:50.053737 3878 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:32:50.053756 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:50.053757 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:32:50.053776 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 25.753µs I0326 23:32:50.067156 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:50.317441 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:50.355168 3878 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-hschh] creating logical port kube-system_global-pull-secret-syncer-hschh for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:50.355295 3878 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-hschh I0326 23:32:50.369699 3878 pod.go:62] [kube-system/global-pull-secret-syncer-hschh] pod update took 14.427069ms I0326 23:32:50.369722 3878 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-hschh] addLogicalPort annotation time took 14.458731ms I0326 23:32:50.370862 3878 pods.go:271] [kube-system/global-pull-secret-syncer-hschh] addLogicalPort took 15.712068ms, libovsdb time 873.503µs I0326 23:32:50.744223 3878 cni.go:403] [kube-system/global-pull-secret-syncer-hschh a33d4c599fa5bf922389a3f4520eb32097bf9327bdac467a44e6e101ffe2e844 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-hschh a33d4c599fa5bf922389a3f4520eb32097bf9327bdac467a44e6e101ffe2e844 network default NAD default] I0326 23:32:50.752761 3878 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-hschh, hostIfaceName: a33d4c599fa5bf9, network: default, NAD default, SandboxID: "a33d4c599fa5bf922389a3f4520eb32097bf9327bdac467a44e6e101ffe2e844", PCI device ID: , UID: "49ed60ab-23a5-441e-8296-627e6f76d0b8", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0326 23:32:50.795136 3878 cni.go:424] [kube-system/global-pull-secret-syncer-hschh a33d4c599fa5bf922389a3f4520eb32097bf9327bdac467a44e6e101ffe2e844 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-hschh a33d4c599fa5bf922389a3f4520eb32097bf9327bdac467a44e6e101ffe2e844 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:e7:fc:fb:e2:33\",\"name\":\"a33d4c599fa5bf9\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cbd94ad7-73da-4041-b9d1-fce0a7f94265\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:32:52.255830 3878 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-zpjxf] creating logical port openshift-insights_insights-operator-5577457d66-zpjxf for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:52.255944 3878 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-zpjxf I0326 23:32:52.268904 3878 pod.go:62] [openshift-insights/insights-operator-5577457d66-zpjxf] pod update took 12.988928ms I0326 23:32:52.268928 3878 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-zpjxf] addLogicalPort annotation time took 13.022528ms I0326 23:32:52.270377 3878 pods.go:271] [openshift-insights/insights-operator-5577457d66-zpjxf] addLogicalPort took 14.565784ms, libovsdb time 1.059656ms I0326 23:32:52.356517 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-h4ldl for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:52.356615 3878 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl I0326 23:32:52.369593 3878 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl] pod update took 12.99984ms I0326 23:32:52.369660 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl] addLogicalPort annotation time took 13.079268ms I0326 23:32:52.370942 3878 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl] addLogicalPort took 14.44439ms, libovsdb time 885.949µs I0326 23:32:52.456523 3878 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-9cj8p] creating logical port openshift-console-operator_console-operator-8777bd9bb-9cj8p for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:52.456682 3878 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-9cj8p I0326 23:32:52.469928 3878 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-9cj8p] pod update took 13.277417ms I0326 23:32:52.469959 3878 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-9cj8p] addLogicalPort annotation time took 13.317778ms I0326 23:32:52.471349 3878 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-9cj8p] addLogicalPort took 14.84735ms, libovsdb time 1.026052ms I0326 23:32:52.563328 3878 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-5wbmn for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:52.563467 3878 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn I0326 23:32:52.576463 3878 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn] pod update took 13.025012ms I0326 23:32:52.576486 3878 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn] addLogicalPort annotation time took 13.048051ms I0326 23:32:52.577649 3878 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn] addLogicalPort took 14.33598ms, libovsdb time 874.422µs I0326 23:32:52.655804 3878 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-86h6c for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:52.655900 3878 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c I0326 23:32:52.669269 3878 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c] pod update took 13.388336ms I0326 23:32:52.669301 3878 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c] addLogicalPort annotation time took 13.424745ms I0326 23:32:52.670580 3878 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c] addLogicalPort took 14.798691ms, libovsdb time 948.014µs I0326 23:32:52.958677 3878 cni.go:403] [openshift-insights/insights-operator-5577457d66-zpjxf a4ecc65b4c55d0c37a0d0f0e9fa3010e48fbf1ee04a5030bdbc4a1a92f91b22b network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-zpjxf a4ecc65b4c55d0c37a0d0f0e9fa3010e48fbf1ee04a5030bdbc4a1a92f91b22b network default NAD default] I0326 23:32:52.968534 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-zpjxf, hostIfaceName: a4ecc65b4c55d0c, network: default, NAD default, SandboxID: "a4ecc65b4c55d0c37a0d0f0e9fa3010e48fbf1ee04a5030bdbc4a1a92f91b22b", PCI device ID: , UID: "0046f7af-105c-44b8-bd9a-7d86f511286a", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0326 23:32:53.016989 3878 cni.go:424] [openshift-insights/insights-operator-5577457d66-zpjxf a4ecc65b4c55d0c37a0d0f0e9fa3010e48fbf1ee04a5030bdbc4a1a92f91b22b network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-zpjxf a4ecc65b4c55d0c37a0d0f0e9fa3010e48fbf1ee04a5030bdbc4a1a92f91b22b network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:12:2f:d4:f7:c8\",\"name\":\"a4ecc65b4c55d0c\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f3df193d-c5aa-4f6d-ac06-e7f713b22ef9\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:32:53.055464 3878 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c acdd12b1a8f790c92318f4390f57efc14c1ba18a636aed954f38db8fe481986d network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c acdd12b1a8f790c92318f4390f57efc14c1ba18a636aed954f38db8fe481986d network default NAD default] I0326 23:32:53.066825 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-86h6c, hostIfaceName: acdd12b1a8f790c, network: default, NAD default, SandboxID: "acdd12b1a8f790c92318f4390f57efc14c1ba18a636aed954f38db8fe481986d", PCI device ID: , UID: "99de914a-7cc3-48ac-ab11-526d04a05500", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0326 23:32:53.109766 3878 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c acdd12b1a8f790c92318f4390f57efc14c1ba18a636aed954f38db8fe481986d network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-86h6c acdd12b1a8f790c92318f4390f57efc14c1ba18a636aed954f38db8fe481986d network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:bd:a6:c6:48:4c\",\"name\":\"acdd12b1a8f790c\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aaefbad3-9199-4610-b584-2e691b2acfaf\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:32:53.157514 3878 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-9cj8p 342ebd23ff5e072903dfefc4a5f68b854e725fd527bc12e27e27c9560a7f1eea network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-9cj8p 342ebd23ff5e072903dfefc4a5f68b854e725fd527bc12e27e27c9560a7f1eea network default NAD default] I0326 23:32:53.169086 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-9cj8p, hostIfaceName: 342ebd23ff5e072, network: default, NAD default, SandboxID: "342ebd23ff5e072903dfefc4a5f68b854e725fd527bc12e27e27c9560a7f1eea", PCI device ID: , UID: "39c15bcb-5019-46b1-8f97-b13bb232478f", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0326 23:32:53.217910 3878 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-9cj8p 342ebd23ff5e072903dfefc4a5f68b854e725fd527bc12e27e27c9560a7f1eea network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-9cj8p 342ebd23ff5e072903dfefc4a5f68b854e725fd527bc12e27e27c9560a7f1eea network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:db:db:0c:0c:16\",\"name\":\"342ebd23ff5e072\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78191731-1e95-42d1-a1e5-3e87ebd254a6\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:32:54.226766 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:32:57.171006 3878 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-6z8lb for pod on switch ip-10-0-2-151.ec2.internal I0326 23:32:57.171099 3878 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb I0326 23:32:57.186244 3878 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb] pod update took 15.173873ms I0326 23:32:57.186266 3878 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb] addLogicalPort annotation time took 15.200595ms I0326 23:32:57.187528 3878 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb] addLogicalPort took 16.537389ms, libovsdb time 883.102µs I0326 23:32:58.000262 3878 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0326 23:32:58.001209 3878 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 918.689µs I0326 23:32:58.046290 3878 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:32:58.457818 3878 namespace.go:98] [openshift-service-ca] adding namespace I0326 23:32:58.458688 3878 namespace.go:102] [openshift-service-ca] adding namespace took 843.664µs I0326 23:32:58.480043 3878 namespace.go:142] [openshift-service-ca] updating namespace W0326 23:32:58.538328 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:32:59.917693 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:00.154039 3878 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-s9gvb] creating logical port openshift-service-ca_service-ca-f59fc589b-s9gvb for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:00.154173 3878 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-s9gvb I0326 23:33:00.166386 3878 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-s9gvb] pod update took 12.236107ms I0326 23:33:00.166470 3878 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-s9gvb] addLogicalPort annotation time took 12.287081ms I0326 23:33:00.167754 3878 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-s9gvb] addLogicalPort took 13.732462ms, libovsdb time 990.299µs I0326 23:33:00.545848 3878 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-s9gvb b70bea2f3d1291cc7a20eed254f33a2bf031e3d2a8932ea55839a8af843b29f6 network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-s9gvb b70bea2f3d1291cc7a20eed254f33a2bf031e3d2a8932ea55839a8af843b29f6 network default NAD default] I0326 23:33:00.555670 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-s9gvb, hostIfaceName: b70bea2f3d1291c, network: default, NAD default, SandboxID: "b70bea2f3d1291cc7a20eed254f33a2bf031e3d2a8932ea55839a8af843b29f6", PCI device ID: , UID: "f26306ac-ec75-43f5-bdf1-8fce45ed8a18", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0326 23:33:00.600016 3878 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-s9gvb b70bea2f3d1291cc7a20eed254f33a2bf031e3d2a8932ea55839a8af843b29f6 network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-s9gvb b70bea2f3d1291cc7a20eed254f33a2bf031e3d2a8932ea55839a8af843b29f6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:e6:37:8e:9d:72\",\"name\":\"b70bea2f3d1291c\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3416ae9-3edb-458f-94da-4091586f7268\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:07.035588 3878 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-1-147.ec2.internal I0326 23:33:07.035615 3878 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-1-147.ec2.internal I0326 23:33:07.035749 3878 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-1-147.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-147.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-147.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-147.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-147.ec2.internal W0326 23:33:07.873001 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:33:08.868591 3878 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn f216be2b0a8daab674b64500562f54dc7b0680130dbbe66b796dd6d215ad2a0d network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn f216be2b0a8daab674b64500562f54dc7b0680130dbbe66b796dd6d215ad2a0d network default NAD default] I0326 23:33:08.878407 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-5wbmn, hostIfaceName: f216be2b0a8daab, network: default, NAD default, SandboxID: "f216be2b0a8daab674b64500562f54dc7b0680130dbbe66b796dd6d215ad2a0d", PCI device ID: , UID: "2c2f50f5-fd04-47d7-a277-e035be39d7c7", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0326 23:33:08.922868 3878 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn f216be2b0a8daab674b64500562f54dc7b0680130dbbe66b796dd6d215ad2a0d network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-5wbmn f216be2b0a8daab674b64500562f54dc7b0680130dbbe66b796dd6d215ad2a0d network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:c7:85:6d:89:76\",\"name\":\"f216be2b0a8daab\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a34e562c-69a4-4750-ba8f-9a0e2aabb186\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:09.551578 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-r7bdt I0326 23:33:09.551614 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-r7bdt I0326 23:33:09.551663 3878 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-r7bdt, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-r7bdt: suppressed error logged: pod openshift-multus/network-metrics-daemon-r7bdt: no pod IPs found I0326 23:33:09.551581 3878 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-pwvv2 I0326 23:33:09.551687 3878 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-pwvv2 I0326 23:33:09.551721 3878 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-pwvv2, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-pwvv2: suppressed error logged: pod openshift-network-diagnostics/network-check-target-pwvv2: no pod IPs found I0326 23:33:11.776522 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:33:11.776551 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 34.208µs I0326 23:33:11.776521 3878 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:33:11.776603 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:13.173042 3878 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb 1096fea50deeb43ee78d107785bfe07a73502bb2c6ca8ad0a783fcddadc605c6 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb 1096fea50deeb43ee78d107785bfe07a73502bb2c6ca8ad0a783fcddadc605c6 network default NAD default] I0326 23:33:13.182374 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-6z8lb, hostIfaceName: 1096fea50deeb43, network: default, NAD default, SandboxID: "1096fea50deeb43ee78d107785bfe07a73502bb2c6ca8ad0a783fcddadc605c6", PCI device ID: , UID: "cdcf411b-8cde-4a4b-a30c-14c22ad2ce6d", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0326 23:33:13.238289 3878 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb 1096fea50deeb43ee78d107785bfe07a73502bb2c6ca8ad0a783fcddadc605c6 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-6z8lb 1096fea50deeb43ee78d107785bfe07a73502bb2c6ca8ad0a783fcddadc605c6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:29:34:f3:37:34\",\"name\":\"1096fea50deeb43\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/17d95479-b1ed-4187-ac4f-e2066eca0de3\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:15.350094 3878 cni.go:403] [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default] I0326 23:33:15.359250 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-5fd5865cf4-sbv9c, hostIfaceName: 88b72030f1342ef, network: default, NAD default, SandboxID: "88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe", PCI device ID: , UID: "de4b981b-6911-4bcd-aa40-d974861e564e", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0326 23:33:15.388563 3878 cni.go:403] [openshift-dns/dns-default-gcmpr f9df7ae189ebb76a1e7ee3b9d24ee07dc164ee7fc120ac5fd943605d9534bfd1 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-gcmpr f9df7ae189ebb76a1e7ee3b9d24ee07dc164ee7fc120ac5fd943605d9534bfd1 network default NAD default] I0326 23:33:15.402562 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-gcmpr, hostIfaceName: f9df7ae189ebb76, network: default, NAD default, SandboxID: "f9df7ae189ebb76a1e7ee3b9d24ee07dc164ee7fc120ac5fd943605d9534bfd1", PCI device ID: , UID: "18ca7ee0-aeb1-4fca-93ae-22cf4e1c70e5", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0326 23:33:15.403759 3878 cni.go:424] [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:80:53:39:ec:fb\",\"name\":\"88b72030f1342ef\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ca1778d4-9321-44bd-83e5-1e0fe4241300\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:15.445739 3878 cni.go:424] [openshift-dns/dns-default-gcmpr f9df7ae189ebb76a1e7ee3b9d24ee07dc164ee7fc120ac5fd943605d9534bfd1 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-gcmpr f9df7ae189ebb76a1e7ee3b9d24ee07dc164ee7fc120ac5fd943605d9534bfd1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:4d:5b:de:09:b2\",\"name\":\"f9df7ae189ebb76\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f86097c-470d-4cf3-a182-35ca10359225\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:17.469661 3878 cni.go:403] [openshift-multus/network-metrics-daemon-4wfbf 1c5f39c0ae828aa7a7319b953a05c138ba0b2ae45a163c0fda4f295d80054828 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-4wfbf 1c5f39c0ae828aa7a7319b953a05c138ba0b2ae45a163c0fda4f295d80054828 network default NAD default] I0326 23:33:17.479095 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-4wfbf, hostIfaceName: 1c5f39c0ae828aa, network: default, NAD default, SandboxID: "1c5f39c0ae828aa7a7319b953a05c138ba0b2ae45a163c0fda4f295d80054828", PCI device ID: , UID: "7fc058f8-f988-4732-adbf-679999231429", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0326 23:33:17.528255 3878 cni.go:424] [openshift-multus/network-metrics-daemon-4wfbf 1c5f39c0ae828aa7a7319b953a05c138ba0b2ae45a163c0fda4f295d80054828 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-4wfbf 1c5f39c0ae828aa7a7319b953a05c138ba0b2ae45a163c0fda4f295d80054828 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:4d:bf:93:22:2a\",\"name\":\"1c5f39c0ae828aa\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85ac14d2-d026-40e3-ab8a-9d0774bb4125\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:20.394874 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:24.343755 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:33:24.349109 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:33:24.349289 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0326 23:33:24.349419 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:33:24.349483 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:33:24.349498 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:33:24.360724 3878 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:24.398863 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:33:24.453176 3878 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-59bb6d878f-phssf IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-59bb6d878f-phssf: no pod IPs found I0326 23:33:24.474028 3878 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-57txv] creating logical port openshift-console_downloads-56d45c498b-57txv for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:24.474125 3878 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-57txv I0326 23:33:24.490124 3878 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-j57pb] creating logical port openshift-insights_insights-runtime-extractor-j57pb for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:24.490234 3878 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-j57pb I0326 23:33:24.504091 3878 pod.go:62] [openshift-console/downloads-56d45c498b-57txv] pod update took 29.992029ms I0326 23:33:24.504120 3878 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-57txv] addLogicalPort annotation time took 30.024947ms I0326 23:33:24.505365 3878 pods.go:271] [openshift-console/downloads-56d45c498b-57txv] addLogicalPort took 31.35624ms, libovsdb time 817.23µs I0326 23:33:24.518591 3878 pod.go:62] [openshift-insights/insights-runtime-extractor-j57pb] pod update took 28.391532ms I0326 23:33:24.518614 3878 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-j57pb] addLogicalPort annotation time took 28.42067ms I0326 23:33:24.519682 3878 pods.go:271] [openshift-insights/insights-runtime-extractor-j57pb] addLogicalPort took 29.60348ms, libovsdb time 782.597µs I0326 23:33:24.542056 3878 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl 66a7a5282c0841c7f7562f89eb7a8d70628ead92246fcea6b9a603379ef747ae network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl 66a7a5282c0841c7f7562f89eb7a8d70628ead92246fcea6b9a603379ef747ae network default NAD default] I0326 23:33:24.551138 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-h4ldl, hostIfaceName: 66a7a5282c0841c, network: default, NAD default, SandboxID: "66a7a5282c0841c7f7562f89eb7a8d70628ead92246fcea6b9a603379ef747ae", PCI device ID: , UID: "3d486a66-deea-4c88-8b39-3b0ccdd24782", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0326 23:33:24.596619 3878 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl 66a7a5282c0841c7f7562f89eb7a8d70628ead92246fcea6b9a603379ef747ae network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-h4ldl 66a7a5282c0841c7f7562f89eb7a8d70628ead92246fcea6b9a603379ef747ae network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:b8:be:17:b0:e2\",\"name\":\"66a7a5282c0841c\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e48580ca-d709-4125-9c02-ae9fb910f535\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:24.834144 3878 cni.go:403] [openshift-console/downloads-56d45c498b-57txv 012b68fabf0e3f2d3e571a64aa3c09adf794a80666eaf82b9304a179991ccbf2 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-57txv 012b68fabf0e3f2d3e571a64aa3c09adf794a80666eaf82b9304a179991ccbf2 network default NAD default] I0326 23:33:24.842887 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-57txv, hostIfaceName: 012b68fabf0e3f2, network: default, NAD default, SandboxID: "012b68fabf0e3f2d3e571a64aa3c09adf794a80666eaf82b9304a179991ccbf2", PCI device ID: , UID: "645ccb63-e8a8-4231-91fe-0cbf81e681a0", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0326 23:33:24.864856 3878 cni.go:403] [openshift-insights/insights-runtime-extractor-j57pb cde90ace41ba4cb585a658cbff3dd29ed3b946ae9ab093660870c86aa0cfab47 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-j57pb cde90ace41ba4cb585a658cbff3dd29ed3b946ae9ab093660870c86aa0cfab47 network default NAD default] I0326 23:33:24.873320 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-j57pb, hostIfaceName: cde90ace41ba4cb, network: default, NAD default, SandboxID: "cde90ace41ba4cb585a658cbff3dd29ed3b946ae9ab093660870c86aa0cfab47", PCI device ID: , UID: "8395edf1-0bc4-4cbd-ac3b-4e1bb56b7c71", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0326 23:33:24.889567 3878 cni.go:424] [openshift-console/downloads-56d45c498b-57txv 012b68fabf0e3f2d3e571a64aa3c09adf794a80666eaf82b9304a179991ccbf2 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-57txv 012b68fabf0e3f2d3e571a64aa3c09adf794a80666eaf82b9304a179991ccbf2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:22:2c:48:ba:6e\",\"name\":\"012b68fabf0e3f2\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d92704e9-7071-4a52-8244-59832d1d560f\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:24.916733 3878 cni.go:424] [openshift-insights/insights-runtime-extractor-j57pb cde90ace41ba4cb585a658cbff3dd29ed3b946ae9ab093660870c86aa0cfab47 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-j57pb cde90ace41ba4cb585a658cbff3dd29ed3b946ae9ab093660870c86aa0cfab47 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:3e:06:11:21:20\",\"name\":\"cde90ace41ba4cb\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a177f3f-2cf5-4617-9e00-e709bfd07cba\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:25.452558 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:33:25.452585 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "ip-10-0-1-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:25.452617 3878 node_tracker.go:235] Node ip-10-0-1-147.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-147.ec2.internal" I0326 23:33:25.452638 3878 node_tracker.go:169] Node ip-10-0-1-147.ec2.internal switch + router changed, syncing services in network "default" I0326 23:33:25.452652 3878 services_controller.go:574] Full service sync requested for network=default W0326 23:33:25.453191 3878 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:25.563220 3878 default_network_controller.go:1008] Node "ip-10-0-1-147.ec2.internal" in remote zone "ip-10-0-1-147.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:33:25.563259 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:33:25.563314 3878 node_tracker.go:169] Node ip-10-0-1-147.ec2.internal switch + router changed, syncing services in network "default" I0326 23:33:25.563333 3878 services_controller.go:574] Full service sync requested for network=default W0326 23:33:25.563872 3878 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:25.563991 3878 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-147.ec2.internal for the network default I0326 23:33:25.565274 3878 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-147.ec2.internal" on network "default" took: 1.298619ms I0326 23:33:26.117293 3878 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:33:26.117334 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:33:26.117354 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 26.575µs I0326 23:33:26.117369 3878 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-147.ec2.internal in network "default" I0326 23:33:26.117424 3878 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 10.0.1.147 to PMTUD blocking rules I0326 23:33:26.117439 3878 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:33:26.117453 3878 node_tracker.go:169] Node ip-10-0-1-147.ec2.internal switch + router changed, syncing services in network "default" I0326 23:33:26.117468 3878 services_controller.go:574] Full service sync requested for network=default W0326 23:33:26.117637 3878 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:33:26.124317 3878 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-147.ec2.internal. Updating no re-route policies I0326 23:33:26.140362 3878 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 10.0.1.147 to PMTUD blocking rules I0326 23:33:26.140381 3878 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-147.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:33:26.187676 3878 base_network_controller_pods.go:487] [default/openshift-console/console-6969bb5f4f-4kk7b] creating logical port openshift-console_console-6969bb5f4f-4kk7b for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:26.187808 3878 kube.go:256] Updating pod openshift-console/console-6969bb5f4f-4kk7b I0326 23:33:26.210280 3878 pod.go:62] [openshift-console/console-6969bb5f4f-4kk7b] pod update took 22.515863ms I0326 23:33:26.210299 3878 base_network_controller_pods.go:937] [default/openshift-console/console-6969bb5f4f-4kk7b] addLogicalPort annotation time took 22.541102ms I0326 23:33:26.211596 3878 pods.go:271] [openshift-console/console-6969bb5f4f-4kk7b] addLogicalPort took 23.941087ms, libovsdb time 912.866µs W0326 23:33:26.232885 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:26.232968 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:26.232982 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:33:26.234173 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:33:26.234187 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0326 23:33:26.246632 3878 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:26.247214 3878 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:26.260501 3878 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:26.260518 3878 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-s6vd6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:26.272480 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:33:26.272504 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:33:26.589632 3878 cni.go:403] [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default] ADD starting CNI request [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default] I0326 23:33:26.601287 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6969bb5f4f-4kk7b, hostIfaceName: 596f57a20224df9, network: default, NAD default, SandboxID: "596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1", PCI device ID: , UID: "9e2532f8-88a0-48a9-b3d2-2d0e29e3cee2", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0326 23:33:26.653626 3878 cni.go:424] [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default] ADD finished CNI request [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:2e:4d:43:0d:ee\",\"name\":\"596f57a20224df9\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f3ac5a0-114b-4987-ae22-797c7e1285dc\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:33:27.222275 3878 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:28.630118 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:33:28.630140 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:33:28.630148 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0326 23:33:29.595419 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:33:29.595463 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 53.077µs W0326 23:33:30.239276 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:33:30.395467 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-9cltv for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:30.395578 3878 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv I0326 23:33:30.422219 3878 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv] pod update took 26.672471ms I0326 23:33:30.422248 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv] addLogicalPort annotation time took 26.708584ms I0326 23:33:30.423472 3878 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv] addLogicalPort took 28.027338ms, libovsdb time 855.217µs W0326 23:33:30.710942 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:33:30.761038 3878 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv 947df937a2b9a5863f081ff5d6a1cfbb1b99af030edd883c42a9583eb4a3cd89 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv 947df937a2b9a5863f081ff5d6a1cfbb1b99af030edd883c42a9583eb4a3cd89 network default NAD default] I0326 23:33:30.772563 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-9cltv, hostIfaceName: 947df937a2b9a58, network: default, NAD default, SandboxID: "947df937a2b9a5863f081ff5d6a1cfbb1b99af030edd883c42a9583eb4a3cd89", PCI device ID: , UID: "762bf166-6f7d-4dea-ba8c-03a220289d70", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0326 23:33:30.839022 3878 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv 947df937a2b9a5863f081ff5d6a1cfbb1b99af030edd883c42a9583eb4a3cd89 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-9cltv 947df937a2b9a5863f081ff5d6a1cfbb1b99af030edd883c42a9583eb4a3cd89 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:af:0f:76:71:82\",\"name\":\"947df937a2b9a58\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/115a7cd5-4308-45b6-98ed-5111c76e385f\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:33:30.962612 3878 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:32.692213 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:33:32.718811 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0326 23:33:33.549649 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:33:36.714948 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:33:37.624470 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:33:40.275358 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:33:40.764321 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0326 23:33:43.108305 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:43.108469 3878 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0326 23:33:43.133479 3878 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 25.043466ms I0326 23:33:43.133505 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 25.076646ms I0326 23:33:43.134844 3878 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 26.557106ms, libovsdb time 958.822µs I0326 23:33:43.479099 3878 cni.go:403] [openshift-monitoring/alertmanager-main-1 9d5450fde9a789c9fb079e21ffed1cc28e18f5c4d2771e94a94597f5b921c53b network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 9d5450fde9a789c9fb079e21ffed1cc28e18f5c4d2771e94a94597f5b921c53b network default NAD default] I0326 23:33:43.488977 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 9d5450fde9a789c, network: default, NAD default, SandboxID: "9d5450fde9a789c9fb079e21ffed1cc28e18f5c4d2771e94a94597f5b921c53b", PCI device ID: , UID: "8ab89ced-475b-4020-8f7e-0d7c662425a2", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0326 23:33:43.541460 3878 cni.go:424] [openshift-monitoring/alertmanager-main-1 9d5450fde9a789c9fb079e21ffed1cc28e18f5c4d2771e94a94597f5b921c53b network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 9d5450fde9a789c9fb079e21ffed1cc28e18f5c4d2771e94a94597f5b921c53b network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:32:50:f7:f0:b8\",\"name\":\"9d5450fde9a789c\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2546bdca-0787-4342-ab6d-980904943a8b\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:44.988703 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-5c98db6c4-b6f5k] creating logical port openshift-monitoring_thanos-querier-5c98db6c4-b6f5k for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:44.988824 3878 kube.go:256] Updating pod openshift-monitoring/thanos-querier-5c98db6c4-b6f5k I0326 23:33:45.017424 3878 pod.go:62] [openshift-monitoring/thanos-querier-5c98db6c4-b6f5k] pod update took 28.644107ms I0326 23:33:45.017458 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-5c98db6c4-b6f5k] addLogicalPort annotation time took 28.684893ms I0326 23:33:45.018927 3878 pods.go:271] [openshift-monitoring/thanos-querier-5c98db6c4-b6f5k] addLogicalPort took 30.240864ms, libovsdb time 1.065178ms I0326 23:33:45.359229 3878 cni.go:403] [openshift-monitoring/thanos-querier-5c98db6c4-b6f5k a0bb21a9a85cd67ef3498bb61efa6c76c88781d8fbf880bf970b6f2fe2d19d3e network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-5c98db6c4-b6f5k a0bb21a9a85cd67ef3498bb61efa6c76c88781d8fbf880bf970b6f2fe2d19d3e network default NAD default] I0326 23:33:45.368320 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5c98db6c4-b6f5k, hostIfaceName: a0bb21a9a85cd67, network: default, NAD default, SandboxID: "a0bb21a9a85cd67ef3498bb61efa6c76c88781d8fbf880bf970b6f2fe2d19d3e", PCI device ID: , UID: "4befb929-c85e-4dcd-abb2-46fe1ac37013", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0326 23:33:45.418160 3878 cni.go:424] [openshift-monitoring/thanos-querier-5c98db6c4-b6f5k a0bb21a9a85cd67ef3498bb61efa6c76c88781d8fbf880bf970b6f2fe2d19d3e network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-5c98db6c4-b6f5k a0bb21a9a85cd67ef3498bb61efa6c76c88781d8fbf880bf970b6f2fe2d19d3e network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:a8:84:7d:29:cb\",\"name\":\"a0bb21a9a85cd67\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4d92e8aa-3918-49f3-913d-c5fbe1f945df\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:33:45.771180 3878 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:45.771204 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:45.771212 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:33:46.437308 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-68998dc46d-b62bl] creating logical port openshift-monitoring_metrics-server-68998dc46d-b62bl for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:46.437477 3878 kube.go:256] Updating pod openshift-monitoring/metrics-server-68998dc46d-b62bl I0326 23:33:46.460309 3878 pod.go:62] [openshift-monitoring/metrics-server-68998dc46d-b62bl] pod update took 22.861783ms I0326 23:33:46.460426 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-68998dc46d-b62bl] addLogicalPort annotation time took 22.985342ms I0326 23:33:46.461882 3878 pods.go:271] [openshift-monitoring/metrics-server-68998dc46d-b62bl] addLogicalPort took 24.59476ms, libovsdb time 1.054612ms I0326 23:33:46.670999 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-lpcxb for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:46.671095 3878 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb I0326 23:33:46.686253 3878 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb] pod update took 15.182496ms I0326 23:33:46.686288 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb] addLogicalPort annotation time took 15.221414ms I0326 23:33:46.687800 3878 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb] addLogicalPort took 16.8142ms, libovsdb time 1.077558ms W0326 23:33:46.782245 3878 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-5c87bcb84c-2nqcd IP address from the namespace address-set, err: pod openshift-console/console-5c87bcb84c-2nqcd: no pod IPs found I0326 23:33:46.831610 3878 cni.go:403] [openshift-monitoring/metrics-server-68998dc46d-b62bl 03851db083e7efca73780d22063c6ec0cedb0808e707a9bbf8a783cc720c0ddd network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-68998dc46d-b62bl 03851db083e7efca73780d22063c6ec0cedb0808e707a9bbf8a783cc720c0ddd network default NAD default] I0326 23:33:46.841822 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-68998dc46d-b62bl, hostIfaceName: 03851db083e7efc, network: default, NAD default, SandboxID: "03851db083e7efca73780d22063c6ec0cedb0808e707a9bbf8a783cc720c0ddd", PCI device ID: , UID: "49ba3b0e-f5e5-4f48-b1d7-a99f7902adbb", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0326 23:33:46.895227 3878 cni.go:424] [openshift-monitoring/metrics-server-68998dc46d-b62bl 03851db083e7efca73780d22063c6ec0cedb0808e707a9bbf8a783cc720c0ddd network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-68998dc46d-b62bl 03851db083e7efca73780d22063c6ec0cedb0808e707a9bbf8a783cc720c0ddd network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:63:be:4d:ec:b6\",\"name\":\"03851db083e7efc\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b7ea87d-2f30-4588-8c90-ff613a11b174\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:33:47.104063 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:47.104085 3878 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:33:47.104091 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:33:47.641006 3878 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb 75cd900aece4a57982d72cfdc328da7434c47aa0f5476485ff297161164dc1f9 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb 75cd900aece4a57982d72cfdc328da7434c47aa0f5476485ff297161164dc1f9 network default NAD default] I0326 23:33:47.651093 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-lpcxb, hostIfaceName: 75cd900aece4a57, network: default, NAD default, SandboxID: "75cd900aece4a57982d72cfdc328da7434c47aa0f5476485ff297161164dc1f9", PCI device ID: , UID: "781118e4-fb1c-4a1f-96f2-afbd18863688", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0326 23:33:47.714214 3878 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb 75cd900aece4a57982d72cfdc328da7434c47aa0f5476485ff297161164dc1f9 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-lpcxb 75cd900aece4a57982d72cfdc328da7434c47aa0f5476485ff297161164dc1f9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:c6:64:ad:db:3c\",\"name\":\"75cd900aece4a57\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/25278099-8d90-42be-9e87-3f5a5d151ed1\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:33:48.294698 3878 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:33:48.294826 3878 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0326 23:33:48.321936 3878 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 27.157069ms I0326 23:33:48.321988 3878 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 27.213973ms I0326 23:33:48.323689 3878 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 29.006949ms, libovsdb time 1.249925ms I0326 23:33:48.704261 3878 cni.go:403] [openshift-monitoring/prometheus-k8s-1 512221d8c3bb4648c6cdfacdea3bc736e4a8a6c6c81380f5c5e716e6fff9ebc7 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 512221d8c3bb4648c6cdfacdea3bc736e4a8a6c6c81380f5c5e716e6fff9ebc7 network default NAD default] I0326 23:33:48.714654 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 512221d8c3bb464, network: default, NAD default, SandboxID: "512221d8c3bb4648c6cdfacdea3bc736e4a8a6c6c81380f5c5e716e6fff9ebc7", PCI device ID: , UID: "999ab78f-8ec1-46dd-8553-05d21fcdb67e", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0326 23:33:48.777032 3878 cni.go:424] [openshift-monitoring/prometheus-k8s-1 512221d8c3bb4648c6cdfacdea3bc736e4a8a6c6c81380f5c5e716e6fff9ebc7 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 512221d8c3bb4648c6cdfacdea3bc736e4a8a6c6c81380f5c5e716e6fff9ebc7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:2f:61:f8:88:53\",\"name\":\"512221d8c3bb464\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9b4039b7-2cd6-4a30-b06c-dd4104087f4c\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:33:48.781518 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:33:49.815011 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:49.815036 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:50.092660 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:33:50.813434 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:50.813456 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:50.813464 3878 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:50.813469 3878 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:54.236822 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:54.236844 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:33:54.567054 3878 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-68997f7bdb-wglmk IP address from the namespace address-set, err: pod openshift-console/console-68997f7bdb-wglmk: no pod IPs found W0326 23:33:54.580695 3878 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-68997f7bdb-msz5m IP address from the namespace address-set, err: pod openshift-console/console-68997f7bdb-msz5m: no pod IPs found I0326 23:33:55.329296 3878 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.329898 3878 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.329938 3878 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.329952 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0326 23:33:55.329969 3878 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:33:55.329979 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:33:56.145357 3878 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.145377 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.145383 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.145420 3878 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:33:56.364096 3878 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:56.364120 3878 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:56.364133 3878 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:56.364145 3878 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390fdmbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:33:56.833245 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.833272 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.833280 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:33:56.833286 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:33:58.850795 3878 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-kg498 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:58.851416 3878 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-kg498 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:59.238004 3878 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:33:59.277427 3878 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:33:59.358379 3878 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-kg498 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:59.358424 3878 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-kg498 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:59.374994 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6969bb5f4f-kg498 I0326 23:33:59.375028 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6969bb5f4f-kg498 I0326 23:34:09.317890 3878 cni.go:403] [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default] DEL starting CNI request [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default] I0326 23:34:09.382457 3878 cni.go:424] [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default] DEL finished CNI request [openshift-console/console-6969bb5f4f-4kk7b 596f57a20224df92912784c6d71ccdabb6b2aa5ab17b44e8cd33ed3c3abd15a1 network default NAD default], result "{}", err I0326 23:34:09.462249 3878 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:09.462304 3878 pods.go:176] Deleting pod: openshift-console/console-6969bb5f4f-4kk7b I0326 23:34:09.463669 3878 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6969bb5f4f-4kk7b, ips: 10.129.0.19 I0326 23:34:09.463709 3878 obj_retry.go:464] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:09.476180 3878 base_network_controller_pods.go:487] [default/openshift-console/console-7b97c46849-gvrf5] creating logical port openshift-console_console-7b97c46849-gvrf5 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:34:09.476297 3878 kube.go:256] Updating pod openshift-console/console-7b97c46849-gvrf5 I0326 23:34:09.491102 3878 pod.go:62] [openshift-console/console-7b97c46849-gvrf5] pod update took 14.839091ms I0326 23:34:09.491183 3878 base_network_controller_pods.go:937] [default/openshift-console/console-7b97c46849-gvrf5] addLogicalPort annotation time took 14.92437ms I0326 23:34:09.492317 3878 pods.go:271] [openshift-console/console-7b97c46849-gvrf5] addLogicalPort took 16.154487ms, libovsdb time 790.996µs W0326 23:34:09.813903 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0326 23:34:09.905372 3878 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:09.905448 3878 obj_retry.go:458] Detected object openshift-console/console-6969bb5f4f-4kk7b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:09.908044 3878 cni.go:403] [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default] ADD starting CNI request [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default] I0326 23:34:09.911114 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6969bb5f4f-4kk7b I0326 23:34:09.911140 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6969bb5f4f-4kk7b I0326 23:34:09.916935 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7b97c46849-gvrf5, hostIfaceName: 3410dcc30d62bed, network: default, NAD default, SandboxID: "3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43", PCI device ID: , UID: "244f1cd4-7a0e-4244-99ad-4732edfcd89e", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0326 23:34:09.965275 3878 cni.go:424] [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default] ADD finished CNI request [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:cc:57:90:bd:49\",\"name\":\"3410dcc30d62bed\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d61834c7-8461-4c53-928a-37c79a0afdd8\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:34:13.464452 3878 cni.go:403] [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default] W0326 23:34:13.515519 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.515539 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.515545 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.515557 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.515562 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.515567 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.533444 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.533465 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:34:13.533471 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:34:13.538435 3878 cni.go:424] [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-5fd5865cf4-sbv9c 88b72030f1342ef7f98cb795f395984da875a4c43c2964d47c0b58d1560a7fbe network default NAD default], result "{}", err I0326 23:34:13.617665 3878 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:13.617706 3878 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5fd5865cf4-sbv9c I0326 23:34:13.619125 3878 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5fd5865cf4-sbv9c, ips: 10.129.0.6 I0326 23:34:13.619161 3878 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:34:13.944895 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0326 23:34:13.944895 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0326 23:34:13.945093 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0326 23:34:13.959647 3878 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:13.959674 3878 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fd5865cf4-sbv9c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:13.966034 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5fd5865cf4-sbv9c I0326 23:34:13.966071 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5fd5865cf4-sbv9c W0326 23:34:14.946443 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0326 23:34:17.431907 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:34:18.421063 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0326 23:34:19.976551 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:34:34.722792 3878 obj_retry.go:464] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:34.723331 3878 obj_retry.go:464] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:35.314695 3878 obj_retry.go:458] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:35.314725 3878 obj_retry.go:458] Detected object openshift-console/console-5c87bcb84c-wj4vj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:35.320743 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5c87bcb84c-wj4vj I0326 23:34:35.320769 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5c87bcb84c-wj4vj W0326 23:34:49.135273 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:34:49.135297 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:35:27.640328 3878 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-jllrk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:27.640915 3878 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-jllrk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:27.961474 3878 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-jllrk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:27.961499 3878 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-jllrk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:27.966856 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b97c46849-jllrk I0326 23:35:27.966883 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b97c46849-jllrk I0326 23:35:35.831904 3878 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:35:38.694322 3878 cni.go:403] [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default] DEL starting CNI request [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default] I0326 23:35:38.771306 3878 cni.go:424] [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default] DEL finished CNI request [openshift-console/console-7b97c46849-gvrf5 3410dcc30d62bed490111d8e362c798d985457ee828a42c1d42905fda275bf43 network default NAD default], result "{}", err I0326 23:35:38.825770 3878 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-gvrf5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:38.825808 3878 pods.go:176] Deleting pod: openshift-console/console-7b97c46849-gvrf5 I0326 23:35:38.827291 3878 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7b97c46849-gvrf5, ips: 10.129.0.26 I0326 23:35:38.827325 3878 obj_retry.go:464] Detected object openshift-console/console-7b97c46849-gvrf5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:39.310251 3878 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-gvrf5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:39.310281 3878 obj_retry.go:458] Detected object openshift-console/console-7b97c46849-gvrf5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:39.316985 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7b97c46849-gvrf5 I0326 23:35:39.317012 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7b97c46849-gvrf5 I0326 23:35:42.844742 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:42.845327 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:35:43.754533 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:35:43.754560 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:37:55.665979 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0326 23:38:11.565028 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0326 23:38:14.645547 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0326 23:38:18.558122 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=105 I0326 23:38:24.677695 3878 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0326 23:38:47.563303 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 W0326 23:39:03.412262 3878 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:39:11.649859 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0326 23:39:14.452051 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:39:14.452119 3878 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-ch8fc IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-ch8fc: no pod IPs found I0326 23:39:23.683330 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:39:37.567283 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=564 I0326 23:39:56.657535 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:40:01.683784 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0326 23:40:03.756653 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:03.757233 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:04.642593 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:04.642636 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:18.672334 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0326 23:40:19.559547 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=72 I0326 23:40:29.661682 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0326 23:40:48.856714 3878 namespace.go:98] [tsf] adding namespace I0326 23:40:48.857586 3878 namespace.go:102] [tsf] adding namespace took 852.917µs I0326 23:40:48.870714 3878 namespace.go:142] [tsf] updating namespace I0326 23:40:48.880078 3878 namespace.go:142] [tsf] updating namespace I0326 23:40:52.859895 3878 namespace.go:98] [konflux-operator] adding namespace I0326 23:40:52.859928 3878 namespace.go:98] [tsf-tas] adding namespace I0326 23:40:52.860751 3878 namespace.go:102] [konflux-operator] adding namespace took 830.408µs I0326 23:40:52.861288 3878 namespace.go:102] [tsf-tas] adding namespace took 1.343394ms I0326 23:40:52.863002 3878 namespace.go:98] [cert-manager-operator] adding namespace I0326 23:40:52.863698 3878 namespace.go:102] [cert-manager-operator] adding namespace took 678.469µs I0326 23:40:52.868269 3878 namespace.go:98] [tsf-tpa] adding namespace I0326 23:40:52.868363 3878 namespace.go:98] [rhtpa-operator] adding namespace I0326 23:40:52.868487 3878 namespace.go:98] [tsf-keycloak] adding namespace I0326 23:40:52.868653 3878 namespace.go:98] [konflux-ui] adding namespace I0326 23:40:52.868976 3878 namespace.go:102] [tsf-tpa] adding namespace took 692.325µs I0326 23:40:52.869600 3878 namespace.go:102] [rhtpa-operator] adding namespace took 1.222152ms I0326 23:40:52.869628 3878 namespace.go:98] [rhbk-operator] adding namespace I0326 23:40:52.870082 3878 namespace.go:102] [rhbk-operator] adding namespace took 445.214µs I0326 23:40:52.870588 3878 namespace.go:102] [tsf-keycloak] adding namespace took 2.084611ms I0326 23:40:52.870934 3878 namespace.go:102] [konflux-ui] adding namespace took 2.270448ms I0326 23:40:52.875930 3878 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:52.893149 3878 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:52.905119 3878 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:52.918479 3878 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:52.930952 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:52.930994 3878 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:52.931108 3878 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:52.948176 3878 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:52.951020 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:52.951077 3878 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:52.951194 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:52.951207 3878 namespace.go:142] [konflux-operator] updating namespace I0326 23:40:52.955856 3878 namespace.go:142] [tsf-tas] updating namespace I0326 23:40:52.958431 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:52.973129 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:52.987868 3878 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:53.002084 3878 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:40:53.016233 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:53.029268 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:53.046905 3878 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:53.062334 3878 namespace.go:142] [rhbk-operator] updating namespace I0326 23:40:53.072358 3878 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:40:53.073631 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:53.088828 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:40:53.912307 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:40:53.923468 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:40:54.067864 3878 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-2-151.ec2.internal I0326 23:40:54.068004 3878 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0326 23:40:54.083835 3878 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 15.869185ms I0326 23:40:54.083868 3878 base_network_controller_pods.go:937] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 15.908284ms I0326 23:40:54.085274 3878 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 17.423483ms, libovsdb time 942.023µs I0326 23:40:54.442919 3878 cni.go:403] [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default] ADD starting CNI request [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default] I0326 23:40:54.451695 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 406292b74a25a19, network: default, NAD default, SandboxID: "406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12", PCI device ID: , UID: "81581667-3747-4e00-9d59-261604c3f829", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0326 23:40:54.498209 3878 cni.go:424] [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default] ADD finished CNI request [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:02:1e:70:64:7c\",\"name\":\"406292b74a25a19\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4cf907e3-fd07-4030-84d1-54187eb25e89\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:41:00.011509 3878 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012109 3878 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012136 3878 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012144 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:00.012159 3878 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:00.012163 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:01.021715 3878 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022257 3878 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022287 3878 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022298 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:01.022310 3878 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:01.022314 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:01.037885 3878 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:01.037913 3878 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:01.037929 3878 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:01.037950 3878 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf7zfdp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048337 3878 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048366 3878 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048381 3878 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:02.048415 3878 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8wmnbf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:06.077114 3878 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077721 3878 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077763 3878 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077775 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:06.077794 3878 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:06.077809 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:07.105079 3878 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.105099 3878 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.105109 3878 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:07.105121 3878 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d15hl5h of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:08.164590 3878 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.165063 3878 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.165093 3878 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.165106 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:08.165120 3878 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.165129 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:08.220506 3878 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.220939 3878 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.220967 3878 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.220975 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:08.220986 3878 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:08.220990 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:09.200991 3878 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.201017 3878 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.201029 3878 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.201045 3878 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7jk88n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255876 3878 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255903 3878 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255917 3878 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:09.255931 3878 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaflmswm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:41:11.885091 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0326 23:41:13.422421 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0326 23:41:13.432531 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:41:19.147107 3878 namespace.go:142] [openshift-operators] updating namespace I0326 23:41:19.158455 3878 namespace.go:142] [openshift-operators] updating namespace W0326 23:41:20.953295 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.106442 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.174608 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.217653 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:41:21.232384 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:41:22.519060 3878 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.519635 3878 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.519670 3878 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.519678 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:41:22.519690 3878 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:41:22.519696 3878 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:41:23.593665 3878 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:23.593696 3878 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:23.593707 3878 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:23.593720 3878 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mv9lg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:26.369335 3878 namespace.go:142] [openshift-operators] updating namespace I0326 23:41:35.467500 3878 namespace.go:98] [trusted-artifact-signer] adding namespace I0326 23:41:35.468453 3878 namespace.go:102] [trusted-artifact-signer] adding namespace took 928.907µs I0326 23:41:35.494784 3878 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:35.541384 3878 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:35.646009 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0326 23:41:36.065336 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:41:41.327504 3878 namespace.go:98] [openshift-pipelines] adding namespace I0326 23:41:41.328346 3878 namespace.go:102] [openshift-pipelines] adding namespace took 819.294µs I0326 23:41:41.348098 3878 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:41:42.015882 3878 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:42.147186 3878 namespace.go:98] [cert-manager] adding namespace I0326 23:41:42.148102 3878 namespace.go:102] [cert-manager] adding namespace took 894.121µs I0326 23:41:42.181415 3878 namespace.go:142] [cert-manager] updating namespace I0326 23:41:42.202760 3878 namespace.go:142] [cert-manager] updating namespace I0326 23:41:42.412451 3878 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:42.628608 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:42.815999 3878 namespace.go:142] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] updating namespace I0326 23:41:43.207180 3878 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:43.408071 3878 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:43.603422 3878 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:43.803341 3878 namespace.go:142] [tsf] updating namespace I0326 23:41:44.040027 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:44.224287 3878 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:44.444060 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:44.550368 3878 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:44.561205 3878 namespace.go:142] [default] updating namespace I0326 23:41:44.573856 3878 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:44.582544 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:44.591915 3878 namespace.go:142] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] updating namespace I0326 23:41:44.601387 3878 namespace.go:142] [openshift] updating namespace I0326 23:41:44.625234 3878 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:44.634360 3878 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:44.642311 3878 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:44.651361 3878 namespace.go:142] [tsf] updating namespace I0326 23:41:44.665434 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:44.677340 3878 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:44.690279 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:44.795105 3878 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:41:44.889828 3878 namespace.go:142] [default] updating namespace I0326 23:41:45.013137 3878 namespace.go:142] [konflux-operator] updating namespace I0326 23:41:45.098945 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:41:45.190138 3878 namespace.go:142] [open-cluster-management-2p9n1cflvtbbv11qgga665kmjqjciu59] updating namespace I0326 23:41:45.291730 3878 namespace.go:142] [openshift] updating namespace I0326 23:41:45.392515 3878 namespace.go:142] [rhbk-operator] updating namespace I0326 23:41:45.491742 3878 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:41:45.592093 3878 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:41:45.700570 3878 namespace.go:142] [tsf] updating namespace I0326 23:41:45.790605 3878 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:41:45.895808 3878 namespace.go:142] [tsf-tas] updating namespace I0326 23:41:45.991708 3878 namespace.go:142] [tsf-tpa] updating namespace I0326 23:41:46.412544 3878 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:41:46.921069 3878 namespace.go:142] [cert-manager] updating namespace I0326 23:41:47.213345 3878 namespace.go:142] [cert-manager] updating namespace I0326 23:41:47.312516 3878 namespace.go:142] [cert-manager] updating namespace W0326 23:41:48.136321 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:41:48.136343 3878 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:41:48.887643 3878 cni.go:403] [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default] DEL starting CNI request [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default] I0326 23:41:48.960639 3878 cni.go:424] [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default] DEL finished CNI request [tsf/test-tsf-subscriptions 406292b74a25a191c94e03a1fc3fe10ce238a358daf8b8635e303b3e992b9b12 network default NAD default], result "{}", err I0326 23:41:49.050366 3878 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:49.050429 3878 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0326 23:41:49.051829 3878 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.129.0.27 I0326 23:41:49.051864 3878 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:41:49.083707 3878 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:49.083729 3878 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:49.093694 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0326 23:41:49.093727 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0326 23:41:51.071836 3878 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-2-151.ec2.internal I0326 23:41:51.071952 3878 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0326 23:41:51.085781 3878 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 13.874358ms I0326 23:41:51.085802 3878 base_network_controller_pods.go:937] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 13.89726ms I0326 23:41:51.087078 3878 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 15.257358ms, libovsdb time 918.824µs I0326 23:41:51.455873 3878 cni.go:403] [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default] ADD starting CNI request [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default] I0326 23:41:51.466977 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: b87b508b70f7270, network: default, NAD default, SandboxID: "b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a", PCI device ID: , UID: "89b037c9-6687-42f5-b232-cfda8eed0848", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0326 23:41:51.513458 3878 cni.go:424] [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default] ADD finished CNI request [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:75:6e:85:b2:2f\",\"name\":\"b87b508b70f7270\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e5f5f29-8d24-426e-8f31-5346594f8a09\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:41:53.945510 3878 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-q44l4 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:41:53.945693 3878 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4 I0326 23:41:53.960668 3878 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4] pod update took 15.003449ms I0326 23:41:53.960696 3878 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4] addLogicalPort annotation time took 15.037394ms I0326 23:41:53.962362 3878 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4] addLogicalPort took 16.87235ms, libovsdb time 1.157021ms I0326 23:41:54.798314 3878 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-lnnbd for pod on switch ip-10-0-2-151.ec2.internal I0326 23:41:54.798415 3878 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd I0326 23:41:54.824087 3878 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd] pod update took 25.707514ms I0326 23:41:54.824110 3878 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd] addLogicalPort annotation time took 25.736501ms I0326 23:41:54.825695 3878 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd] addLogicalPort took 27.399883ms, libovsdb time 1.059159ms I0326 23:41:57.576679 3878 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd 9dfc418f9c02169bea9b65a301e171256fb6f4095e523d4b729ea6d4abde79e9 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd 9dfc418f9c02169bea9b65a301e171256fb6f4095e523d4b729ea6d4abde79e9 network default NAD default] I0326 23:41:57.585352 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-lnnbd, hostIfaceName: 9dfc418f9c02169, network: default, NAD default, SandboxID: "9dfc418f9c02169bea9b65a301e171256fb6f4095e523d4b729ea6d4abde79e9", PCI device ID: , UID: "b77cab07-01ac-4aa7-a87b-2cc3ce55f465", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0326 23:41:57.634281 3878 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd 9dfc418f9c02169bea9b65a301e171256fb6f4095e523d4b729ea6d4abde79e9 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-lnnbd 9dfc418f9c02169bea9b65a301e171256fb6f4095e523d4b729ea6d4abde79e9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:ea:d0:76:2a:1b\",\"name\":\"9dfc418f9c02169\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad6c5b6a-c9b1-4d5a-98c4-c9e3f76a4777\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:41:58.272333 3878 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:41:58.272353 3878 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:41:58.272359 3878 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:41:58.291314 3878 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:41:58.291336 3878 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:41:58.291341 3878 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:41:59.109903 3878 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:41:59.109920 3878 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:41:59.109926 3878 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:41:59.109930 3878 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:41:59.131646 3878 cni.go:403] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4 c39f9b9f928af180d6d54e97380af38b320e45fb52ceac8f753e761db8881db1 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4 c39f9b9f928af180d6d54e97380af38b320e45fb52ceac8f753e761db8881db1 network default NAD default] I0326 23:41:59.141222 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-q44l4, hostIfaceName: c39f9b9f928af18, network: default, NAD default, SandboxID: "c39f9b9f928af180d6d54e97380af38b320e45fb52ceac8f753e761db8881db1", PCI device ID: , UID: "757c28c9-4c22-476e-900a-440bbd714112", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0326 23:41:59.193222 3878 cni.go:424] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4 c39f9b9f928af180d6d54e97380af38b320e45fb52ceac8f753e761db8881db1 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-q44l4 c39f9b9f928af180d6d54e97380af38b320e45fb52ceac8f753e761db8881db1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:9a:31:f5:23:85\",\"name\":\"c39f9b9f928af18\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c6968984-42da-44ce-82ad-9fcb4368d1bb\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:42:00.990951 3878 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0326 23:42:05.140043 3878 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0326 23:42:11.560471 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0326 23:42:12.548834 3878 egressfirewall.go:628] Syncing node "ip-10-0-1-147.ec2.internal" for egress firewall I0326 23:42:12.548843 3878 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-147.ec2.internal I0326 23:42:12.548880 3878 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-147.ec2.internal: 43.131µs I0326 23:42:16.399433 3878 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-mvjpf for pod on switch ip-10-0-2-151.ec2.internal I0326 23:42:16.399558 3878 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf I0326 23:42:16.414763 3878 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf] pod update took 15.228513ms I0326 23:42:16.414794 3878 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf] addLogicalPort annotation time took 15.265967ms I0326 23:42:16.416089 3878 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf] addLogicalPort took 16.692834ms, libovsdb time 917.963µs I0326 23:42:16.624541 3878 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-l4phw for pod on switch ip-10-0-2-151.ec2.internal I0326 23:42:16.624672 3878 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw I0326 23:42:16.638786 3878 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw] pod update took 14.139456ms I0326 23:42:16.638816 3878 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw] addLogicalPort annotation time took 14.174949ms I0326 23:42:16.640282 3878 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw] addLogicalPort took 15.758925ms, libovsdb time 1.007744ms I0326 23:42:16.781557 3878 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf d653ada7e24cbcb7c5849a1c6817602a530f6e6e03fbeeeae507a000b6f55a19 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf d653ada7e24cbcb7c5849a1c6817602a530f6e6e03fbeeeae507a000b6f55a19 network default NAD default] I0326 23:42:16.790872 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-mvjpf, hostIfaceName: d653ada7e24cbcb, network: default, NAD default, SandboxID: "d653ada7e24cbcb7c5849a1c6817602a530f6e6e03fbeeeae507a000b6f55a19", PCI device ID: , UID: "38410325-2ab2-471e-845b-9e1d97091342", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0326 23:42:16.839592 3878 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf d653ada7e24cbcb7c5849a1c6817602a530f6e6e03fbeeeae507a000b6f55a19 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-mvjpf d653ada7e24cbcb7c5849a1c6817602a530f6e6e03fbeeeae507a000b6f55a19 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:8f:fd:7d:b7:c4\",\"name\":\"d653ada7e24cbcb\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f6ff42c2-acaf-4fca-b00b-8abbee7f40b7\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:42:16.994190 3878 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw e22edb92bbdbbdd602f1918d8f6fed4517120b739451e0d499b3ae5b8f9dad8b network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw e22edb92bbdbbdd602f1918d8f6fed4517120b739451e0d499b3ae5b8f9dad8b network default NAD default] I0326 23:42:17.002846 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-l4phw, hostIfaceName: e22edb92bbdbbdd, network: default, NAD default, SandboxID: "e22edb92bbdbbdd602f1918d8f6fed4517120b739451e0d499b3ae5b8f9dad8b", PCI device ID: , UID: "1d242ea3-00cf-4169-8d4e-e3143f0105c2", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0326 23:42:17.049598 3878 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw e22edb92bbdbbdd602f1918d8f6fed4517120b739451e0d499b3ae5b8f9dad8b network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-l4phw e22edb92bbdbbdd602f1918d8f6fed4517120b739451e0d499b3ae5b8f9dad8b network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:02:11:09:a7:37\",\"name\":\"e22edb92bbdbbdd\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/88bcd5c0-4f59-40c7-8f8e-8c10583854d7\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:42:20.085140 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0326 23:42:21.965234 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0326 23:42:29.111602 3878 cni.go:403] [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default] DEL starting CNI request [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default] I0326 23:42:29.192091 3878 cni.go:424] [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default] DEL finished CNI request [tsf/test-tsf-infrastructure b87b508b70f72704090c6478c1cff7ba1d4dc0b5d7a16fc73efa5b854ce6c15a network default NAD default], result "{}", err I0326 23:42:29.245785 3878 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:29.245826 3878 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0326 23:42:29.247275 3878 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.129.0.28 I0326 23:42:29.247310 3878 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:29.266822 3878 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:29.266849 3878 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:29.273858 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0326 23:42:29.273887 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0326 23:42:30.370213 3878 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-151.ec2.internal I0326 23:42:30.370308 3878 kube.go:256] Updating pod tsf/test-tsf-iam I0326 23:42:30.389814 3878 pod.go:62] [tsf/test-tsf-iam] pod update took 19.530152ms I0326 23:42:30.389838 3878 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 19.559049ms I0326 23:42:30.391173 3878 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 20.981384ms, libovsdb time 1.008602ms I0326 23:42:30.434410 3878 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:42:30.434511 3878 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0326 23:42:30.452224 3878 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 17.740242ms I0326 23:42:30.452245 3878 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 17.767081ms I0326 23:42:30.453495 3878 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 19.116484ms, libovsdb time 829.61µs I0326 23:42:30.734545 3878 cni.go:403] [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default] I0326 23:42:30.743504 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 989d3c0b6f1b11a, network: default, NAD default, SandboxID: "989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd", PCI device ID: , UID: "815ce652-f132-4e52-baf6-3515a9c429fc", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0326 23:42:30.793810 3878 cni.go:424] [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:29:2a:5e:c6:35\",\"name\":\"989d3c0b6f1b11a\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7258cd7-6ddf-4acb-90ad-474da07a0a7d\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:42:30.797221 3878 cni.go:403] [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default] I0326 23:42:30.805780 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 41e0b3fd5774133, network: default, NAD default, SandboxID: "41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a", PCI device ID: , UID: "db2bc168-737f-4664-9455-2f229d7452bd", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0326 23:42:30.854022 3878 cni.go:424] [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:c0:df:63:98:f2\",\"name\":\"41e0b3fd5774133\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/baffcfe8-1348-489c-8033-18be1b854326\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:42:31.072381 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0326 23:42:36.153479 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:42:36.153501 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:42:36.155162 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:42:37.404556 3878 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:42:55.625920 3878 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:42:55.625940 3878 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:42:55.625944 3878 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:42:56.089728 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:42:56.093201 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:42:56.093220 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:42:57.381857 3878 cni.go:403] [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default] I0326 23:42:57.465884 3878 cni.go:424] [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 41e0b3fd5774133277252c1f81d740e4b2f3dc7eb9f4f49016542d46693f588a network default NAD default], result "{}", err I0326 23:42:57.518409 3878 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:57.518476 3878 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0326 23:42:57.519839 3878 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.129.0.34 I0326 23:42:57.519872 3878 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:42:58.260030 3878 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:58.260054 3878 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:42:58.267816 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:42:58.267842 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0326 23:42:58.297881 3878 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:42:58.297981 3878 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0326 23:42:58.311929 3878 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 13.972233ms I0326 23:42:58.311955 3878 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 14.004478ms I0326 23:42:58.313173 3878 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 15.309301ms, libovsdb time 813.189µs I0326 23:42:58.660444 3878 cni.go:403] [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default] I0326 23:42:58.669331 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=5 I0326 23:42:58.670558 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: e3491bc38effe1f, network: default, NAD default, SandboxID: "e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919", PCI device ID: , UID: "f71592b1-1e76-4e2a-931e-695253ae7eb4", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0326 23:42:58.718138 3878 cni.go:424] [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:e9:bf:fb:8e:65\",\"name\":\"e3491bc38effe1f\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42304566-6f33-4e8b-9529-44398f2f9fc7\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:43:00.285974 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:00.286101 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:00.286039 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:43:13.983240 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:13.983264 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:13.983362 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:43:20.713298 3878 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-7bwv6] creating logical port tsf-keycloak_tsf-iam-7bwv6 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:43:20.713449 3878 kube.go:256] Updating pod tsf-keycloak/tsf-iam-7bwv6 I0326 23:43:20.729263 3878 pod.go:62] [tsf-keycloak/tsf-iam-7bwv6] pod update took 15.84205ms I0326 23:43:20.729290 3878 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-7bwv6] addLogicalPort annotation time took 15.89438ms I0326 23:43:20.730621 3878 pods.go:271] [tsf-keycloak/tsf-iam-7bwv6] addLogicalPort took 17.340371ms, libovsdb time 896.974µs I0326 23:43:21.086631 3878 cni.go:403] [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default] I0326 23:43:21.095975 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-7bwv6, hostIfaceName: 98de05b6229e52e, network: default, NAD default, SandboxID: "98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27", PCI device ID: , UID: "1191e05e-26c2-44fc-aa63-227f300d1bd6", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0326 23:43:21.145859 3878 cni.go:424] [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:29:78:33:20:f8\",\"name\":\"98de05b6229e52e\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/37b1de6a-c6c0-4d76-9876-f9c15145d6ee\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:43:28.793570 3878 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0326 23:43:31.475145 3878 cni.go:403] [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default] I0326 23:43:31.565342 3878 cni.go:424] [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 989d3c0b6f1b11a388598b9dc73445155d6a7f0cfe5c4fa2547352edbc4ec4fd network default NAD default], result "{}", err I0326 23:43:31.640544 3878 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:31.640592 3878 pods.go:176] Deleting pod: tsf/test-tsf-iam I0326 23:43:31.642111 3878 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.129.0.33 I0326 23:43:31.642142 3878 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:38.505714 3878 cni.go:403] [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default] I0326 23:43:38.585151 3878 cni.go:424] [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-7bwv6 98de05b6229e52edd585805bcc81576330cbcc81d69c49eea107ec63b521ec27 network default NAD default], result "{}", err I0326 23:43:38.637616 3878 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:38.637653 3878 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-7bwv6 I0326 23:43:38.639049 3878 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-7bwv6, ips: 10.129.0.36 I0326 23:43:38.639074 3878 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:39.561715 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=91 I0326 23:43:39.665030 3878 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:39.665057 3878 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-7bwv6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:39.888690 3878 cni.go:403] [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default] I0326 23:43:39.956384 3878 cni.go:424] [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 e3491bc38effe1fcc2f72fef2a9d697eaee0584985cd8b8b5665a0455310c919 network default NAD default], result "{}", err I0326 23:43:40.016775 3878 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:40.016816 3878 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0326 23:43:40.018091 3878 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.129.0.35 I0326 23:43:40.018127 3878 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:43:40.521635 3878 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:40.521660 3878 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:43:40.527266 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:43:40.527303 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0326 23:43:40.569705 3878 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:43:40.569796 3878 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0326 23:43:40.593061 3878 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 23.289879ms I0326 23:43:40.593086 3878 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 23.320192ms I0326 23:43:40.594352 3878 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 24.663192ms, libovsdb time 852.966µs I0326 23:43:40.874088 3878 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-6g9ws for pod on switch ip-10-0-2-151.ec2.internal I0326 23:43:40.874175 3878 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws I0326 23:43:40.891158 3878 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws] pod update took 17.002516ms I0326 23:43:40.891189 3878 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws] addLogicalPort annotation time took 17.039176ms I0326 23:43:40.892578 3878 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws] addLogicalPort took 18.485478ms, libovsdb time 913.978µs I0326 23:43:40.932707 3878 cni.go:403] [tsf-keycloak/keycloak-0 6707c96e78dc51c826000a14f0da27d0da3cb7e61111c9d234b4c6fdf7f259b7 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 6707c96e78dc51c826000a14f0da27d0da3cb7e61111c9d234b4c6fdf7f259b7 network default NAD default] I0326 23:43:40.941856 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 6707c96e78dc51c, network: default, NAD default, SandboxID: "6707c96e78dc51c826000a14f0da27d0da3cb7e61111c9d234b4c6fdf7f259b7", PCI device ID: , UID: "68230c18-821b-4849-8450-c5b0d5acb7d5", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0326 23:43:40.990228 3878 cni.go:424] [tsf-keycloak/keycloak-0 6707c96e78dc51c826000a14f0da27d0da3cb7e61111c9d234b4c6fdf7f259b7 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 6707c96e78dc51c826000a14f0da27d0da3cb7e61111c9d234b4c6fdf7f259b7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:bc:f3:91:dc:43\",\"name\":\"6707c96e78dc51c\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32d568d8-23bf-4e9b-9da1-e70eb10bab96\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:43:41.845289 3878 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws 4a0997712767a97e683faaa842c7de82ebf0dad7ebba02199741e7dbd6d09a69 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws 4a0997712767a97e683faaa842c7de82ebf0dad7ebba02199741e7dbd6d09a69 network default NAD default] I0326 23:43:41.856451 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-6g9ws, hostIfaceName: 4a0997712767a97, network: default, NAD default, SandboxID: "4a0997712767a97e683faaa842c7de82ebf0dad7ebba02199741e7dbd6d09a69", PCI device ID: , UID: "9794e3d7-f199-4170-aa5e-1a0d49d861a1", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0326 23:43:41.909506 3878 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws 4a0997712767a97e683faaa842c7de82ebf0dad7ebba02199741e7dbd6d09a69 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-6g9ws 4a0997712767a97e683faaa842c7de82ebf0dad7ebba02199741e7dbd6d09a69 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:8e:da:7d:5c:63\",\"name\":\"4a0997712767a97\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5202edba-78ef-48a7-aa1f-54811132a50b\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:43:42.544283 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:43:42.547739 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:42.547759 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:57.346167 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:43:57.352310 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:43:57.352329 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:44:18.648164 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0326 23:44:29.697699 3878 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:44:31.792425 3878 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:31.792453 3878 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:31.797718 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:44:31.797744 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:44:31.943315 3878 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-151.ec2.internal I0326 23:44:31.943459 3878 kube.go:256] Updating pod tsf/test-tsf-iam I0326 23:44:31.956725 3878 pod.go:62] [tsf/test-tsf-iam] pod update took 13.298509ms I0326 23:44:31.956746 3878 base_network_controller_pods.go:937] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 13.354568ms I0326 23:44:31.957956 3878 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 14.666342ms, libovsdb time 846.581µs I0326 23:44:32.303733 3878 cni.go:403] [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default] ADD starting CNI request [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default] I0326 23:44:32.312473 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 806e9d5fb0b3bd2, network: default, NAD default, SandboxID: "806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04", PCI device ID: , UID: "3cdd5c85-9b17-40e5-9135-640b3a2e549e", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0326 23:44:32.361126 3878 cni.go:424] [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default] ADD finished CNI request [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:e3:79:4b:de:14\",\"name\":\"806e9d5fb0b3bd2\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b1a05cf-cb3c-43eb-9547-354f94d50a24\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:44:40.842569 3878 cni.go:403] [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default] DEL starting CNI request [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default] I0326 23:44:40.927131 3878 cni.go:424] [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default] DEL finished CNI request [tsf/test-tsf-iam 806e9d5fb0b3bd2d73b14e04ada1e509652183244b75e07736521febeaec8d04 network default NAD default], result "{}", err I0326 23:44:40.979551 3878 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:44:40.979595 3878 pods.go:176] Deleting pod: tsf/test-tsf-iam I0326 23:44:40.980975 3878 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.129.0.39 I0326 23:44:40.981005 3878 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:44:41.002618 3878 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:41.002641 3878 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:44:41.010287 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:44:41.010306 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:44:41.814826 3878 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-2-151.ec2.internal I0326 23:44:41.814951 3878 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0326 23:44:41.830065 3878 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 15.132838ms I0326 23:44:41.830095 3878 base_network_controller_pods.go:937] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 15.169456ms I0326 23:44:41.831679 3878 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 16.86937ms, libovsdb time 1.151557ms I0326 23:44:42.182152 3878 cni.go:403] [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default] ADD starting CNI request [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default] I0326 23:44:42.192670 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: f3954a0636d0a63, network: default, NAD default, SandboxID: "f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e", PCI device ID: , UID: "92677a10-0d24-47c7-aaae-27d5c2bd537e", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0326 23:44:42.240364 3878 cni.go:424] [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default] ADD finished CNI request [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:17:20:49:b6:da\",\"name\":\"f3954a0636d0a63\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6383d939-92f3-40f7-b166-5c3f9b8e85c6\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:44:42.673255 3878 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-q264k-wkbnp] creating logical port tsf-tas_segment-backup-installation-q264k-wkbnp for pod on switch ip-10-0-2-151.ec2.internal I0326 23:44:42.673413 3878 kube.go:256] Updating pod tsf-tas/segment-backup-installation-q264k-wkbnp I0326 23:44:42.684694 3878 pod.go:62] [tsf-tas/segment-backup-installation-q264k-wkbnp] pod update took 11.327456ms I0326 23:44:42.684719 3878 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-q264k-wkbnp] addLogicalPort annotation time took 11.353743ms I0326 23:44:42.685986 3878 pods.go:271] [tsf-tas/segment-backup-installation-q264k-wkbnp] addLogicalPort took 12.752ms, libovsdb time 896.495µs I0326 23:44:43.036500 3878 cni.go:403] [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default] I0326 23:44:43.045609 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-q264k-wkbnp, hostIfaceName: 043f1f773ee788b, network: default, NAD default, SandboxID: "043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b", PCI device ID: , UID: "a44d379f-ea30-426f-b243-c1a519d92491", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0326 23:44:43.097095 3878 cni.go:424] [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:3c:78:67:db:7d\",\"name\":\"043f1f773ee788b\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/643892aa-72e1-4700-b545-9030c8d3f77a\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:44:43.560606 3878 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-95pjt-wbk5x] creating logical port tsf-tas_ctlog-createtree-job-95pjt-wbk5x for pod on switch ip-10-0-2-151.ec2.internal I0326 23:44:43.560719 3878 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-95pjt-wbk5x I0326 23:44:43.575247 3878 pod.go:62] [tsf-tas/ctlog-createtree-job-95pjt-wbk5x] pod update took 14.560392ms I0326 23:44:43.575274 3878 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-95pjt-wbk5x] addLogicalPort annotation time took 14.592015ms I0326 23:44:43.576657 3878 pods.go:271] [tsf-tas/ctlog-createtree-job-95pjt-wbk5x] addLogicalPort took 16.07303ms, libovsdb time 949.475µs I0326 23:44:43.793139 3878 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-68bc499788-fllx6] creating logical port tsf-tas_trillian-logserver-68bc499788-fllx6 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:44:43.793240 3878 kube.go:256] Updating pod tsf-tas/trillian-logserver-68bc499788-fllx6 I0326 23:44:43.813267 3878 pod.go:62] [tsf-tas/trillian-logserver-68bc499788-fllx6] pod update took 20.052106ms I0326 23:44:43.813297 3878 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logserver-68bc499788-fllx6] addLogicalPort annotation time took 20.087936ms I0326 23:44:43.814783 3878 pods.go:271] [tsf-tas/trillian-logserver-68bc499788-fllx6] addLogicalPort took 21.667153ms, libovsdb time 934.327µs I0326 23:44:43.923898 3878 cni.go:403] [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default] I0326 23:44:43.932524 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-95pjt-wbk5x, hostIfaceName: 8dea2e0cb766ccc, network: default, NAD default, SandboxID: "8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6", PCI device ID: , UID: "1e057611-d331-4e1c-94ba-2d61899492ec", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0326 23:44:43.981474 3878 cni.go:424] [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:9b:85:84:53:70\",\"name\":\"8dea2e0cb766ccc\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf908c78-291d-4d16-b077-4b20eaf07817\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:44:44.782174 3878 cni.go:403] [tsf-tas/trillian-logserver-68bc499788-fllx6 4641b9d54ca76d728b2866aaddeabd5a5a533b2d4c6e27735abb25a43821c4dd network default NAD default] ADD starting CNI request [tsf-tas/trillian-logserver-68bc499788-fllx6 4641b9d54ca76d728b2866aaddeabd5a5a533b2d4c6e27735abb25a43821c4dd network default NAD default] I0326 23:44:44.794836 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-68bc499788-fllx6, hostIfaceName: 4641b9d54ca76d7, network: default, NAD default, SandboxID: "4641b9d54ca76d728b2866aaddeabd5a5a533b2d4c6e27735abb25a43821c4dd", PCI device ID: , UID: "8efaf3b5-0287-4f06-8144-378858887030", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0326 23:44:44.867316 3878 cni.go:424] [tsf-tas/trillian-logserver-68bc499788-fllx6 4641b9d54ca76d728b2866aaddeabd5a5a533b2d4c6e27735abb25a43821c4dd network default NAD default] ADD finished CNI request [tsf-tas/trillian-logserver-68bc499788-fllx6 4641b9d54ca76d728b2866aaddeabd5a5a533b2d4c6e27735abb25a43821c4dd network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:ac:8e:93:cb:f0\",\"name\":\"4641b9d54ca76d7\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39c0da82-e0b2-4433-bfb3-8e6e57a9aeab\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:44:46.892253 3878 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:46.892273 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0326 23:44:51.260711 3878 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0326 23:44:55.943125 3878 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0326 23:44:56.994439 3878 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:44:59.986863 3878 cni.go:403] [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default] I0326 23:45:00.077269 3878 cni.go:424] [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-q264k-wkbnp 043f1f773ee788bca22b7dbc9d5a64c342543ec38c75faa1d684cb696e38c41b network default NAD default], result "{}", err I0326 23:45:00.127318 3878 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:00.127360 3878 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-q264k-wkbnp I0326 23:45:00.128754 3878 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-q264k-wkbnp, ips: 10.129.0.41 I0326 23:45:00.128785 3878 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:01.147798 3878 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:01.147824 3878 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-q264k-wkbnp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:45:26.071923 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:45:31.142746 3878 cni.go:403] [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default] DEL starting CNI request [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default] I0326 23:45:31.218472 3878 cni.go:424] [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default] DEL finished CNI request [tsf-tas/test-tsf-tas f3954a0636d0a6341cc3cf97ba7fd09d06980062fd950c698aaf87e641a7e22e network default NAD default], result "{}", err I0326 23:45:31.274030 3878 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:31.274074 3878 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0326 23:45:31.275356 3878 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.129.0.40 I0326 23:45:31.275384 3878 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:31.300198 3878 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:31.300225 3878 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:31.307291 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0326 23:45:31.307321 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0326 23:45:40.289386 3878 obj_retry.go:464] Detected object tsf/patch-tekton-config-vbftd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:40.290043 3878 obj_retry.go:464] Detected object tsf/patch-tekton-config-vbftd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:41.174890 3878 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:41.174921 3878 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:41.283411 3878 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-4gd4n] creating logical port tsf_tsf-tekton-configuration-4gd4n for pod on switch ip-10-0-2-151.ec2.internal I0326 23:45:41.283542 3878 kube.go:256] Updating pod tsf/tsf-tekton-configuration-4gd4n I0326 23:45:41.296511 3878 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:41.297113 3878 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:41.297610 3878 pod.go:62] [tsf/tsf-tekton-configuration-4gd4n] pod update took 14.093328ms I0326 23:45:41.297686 3878 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-4gd4n] addLogicalPort annotation time took 14.172709ms I0326 23:45:41.299013 3878 pods.go:271] [tsf/tsf-tekton-configuration-4gd4n] addLogicalPort took 15.647566ms, libovsdb time 899.005µs I0326 23:45:41.649625 3878 cni.go:403] [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default] I0326 23:45:41.658405 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-4gd4n, hostIfaceName: a0a7a88a1d9e2d1, network: default, NAD default, SandboxID: "a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0", PCI device ID: , UID: "122dac4a-fc0d-4b17-b481-f59b6708826c", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0326 23:45:41.711421 3878 cni.go:424] [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:ac:ef:e7:f1:90\",\"name\":\"a0a7a88a1d9e2d1\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4719fa9c-5158-4f10-bf40-2bcc3c36ebc2\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:45:42.189037 3878 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:42.189063 3878 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wddsf-zzkhg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:44.568586 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=98 I0326 23:45:45.436260 3878 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf] creating logical port tsf-tas_rekor-search-ui-65b86bd7fb-lkbkf for pod on switch ip-10-0-2-151.ec2.internal I0326 23:45:45.436382 3878 kube.go:256] Updating pod tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf I0326 23:45:45.448617 3878 pod.go:62] [tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf] pod update took 12.268617ms I0326 23:45:45.448649 3878 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf] addLogicalPort annotation time took 12.308124ms I0326 23:45:45.450190 3878 pods.go:271] [tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf] addLogicalPort took 13.950969ms, libovsdb time 1.001078ms I0326 23:45:45.566911 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=63 I0326 23:45:45.803776 3878 cni.go:403] [tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf 73770188ecf0cc196e61fa0f215f07ca44a7d48421714325db2890fbe499acea network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf 73770188ecf0cc196e61fa0f215f07ca44a7d48421714325db2890fbe499acea network default NAD default] I0326 23:45:45.813017 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-65b86bd7fb-lkbkf, hostIfaceName: 73770188ecf0cc1, network: default, NAD default, SandboxID: "73770188ecf0cc196e61fa0f215f07ca44a7d48421714325db2890fbe499acea", PCI device ID: , UID: "9f9d0316-f4d5-428c-95b0-945108a052bb", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0326 23:45:45.864482 3878 cni.go:424] [tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf 73770188ecf0cc196e61fa0f215f07ca44a7d48421714325db2890fbe499acea network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-65b86bd7fb-lkbkf 73770188ecf0cc196e61fa0f215f07ca44a7d48421714325db2890fbe499acea network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:ff:a5:61:3c:26\",\"name\":\"73770188ecf0cc1\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c90ca621-7d2f-459d-8e0e-5a734fe78804\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:45:47.227502 3878 cni.go:403] [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default] I0326 23:45:47.298316 3878 cni.go:424] [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-4gd4n a0a7a88a1d9e2d18b0280b023b2bd12bbc38e89731cb799baa4ce3936f9fdec0 network default NAD default], result "{}", err I0326 23:45:47.348321 3878 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-4gd4n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:47.348353 3878 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-4gd4n I0326 23:45:47.349761 3878 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-4gd4n, ips: 10.129.0.44 I0326 23:45:47.349791 3878 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-4gd4n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:47.570563 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=476 I0326 23:45:47.970628 3878 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-7c6df856d7-6gs6q] creating logical port tsf-tas_rekor-server-7c6df856d7-6gs6q for pod on switch ip-10-0-2-151.ec2.internal I0326 23:45:47.970719 3878 kube.go:256] Updating pod tsf-tas/rekor-server-7c6df856d7-6gs6q I0326 23:45:47.983437 3878 pod.go:62] [tsf-tas/rekor-server-7c6df856d7-6gs6q] pod update took 12.739825ms I0326 23:45:47.983462 3878 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-7c6df856d7-6gs6q] addLogicalPort annotation time took 12.76836ms I0326 23:45:47.984860 3878 pods.go:271] [tsf-tas/rekor-server-7c6df856d7-6gs6q] addLogicalPort took 14.251243ms, libovsdb time 932.835µs W0326 23:45:48.197565 3878 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:45:48.370767 3878 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-4gd4n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.370791 3878 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-4gd4n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.425775 3878 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.425799 3878 obj_retry.go:458] Detected object tsf/patch-tekton-config-vbftd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:48.430817 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-vbftd I0326 23:45:48.430838 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-vbftd I0326 23:45:48.571555 3878 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-2-151.ec2.internal I0326 23:45:48.571651 3878 kube.go:256] Updating pod tsf/test-tsf-pipelines I0326 23:45:48.583678 3878 pod.go:62] [tsf/test-tsf-pipelines] pod update took 12.048906ms I0326 23:45:48.583709 3878 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 12.087053ms I0326 23:45:48.585045 3878 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 13.507033ms, libovsdb time 901.622µs I0326 23:45:48.946450 3878 cni.go:403] [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default] I0326 23:45:48.957236 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: 6e22bfe60c5ff03, network: default, NAD default, SandboxID: "6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac", PCI device ID: , UID: "d73c45b2-b8af-430b-80a2-82a74aa04c86", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0326 23:45:49.014516 3878 cni.go:424] [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:a7:cb:96:30:46\",\"name\":\"6e22bfe60c5ff03\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5107652c-e998-48a4-8914-097ce302e911\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:45:51.031861 3878 cni.go:403] [tsf-tas/rekor-server-7c6df856d7-6gs6q 0b1df59207deb6abc0fe0a196d38ab6f516e7b3440843892dbbbcb12646924dd network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-7c6df856d7-6gs6q 0b1df59207deb6abc0fe0a196d38ab6f516e7b3440843892dbbbcb12646924dd network default NAD default] I0326 23:45:51.040747 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-7c6df856d7-6gs6q, hostIfaceName: 0b1df59207deb6a, network: default, NAD default, SandboxID: "0b1df59207deb6abc0fe0a196d38ab6f516e7b3440843892dbbbcb12646924dd", PCI device ID: , UID: "68167150-ae12-4375-b25b-6217b9cdba42", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0326 23:45:51.088263 3878 cni.go:424] [tsf-tas/rekor-server-7c6df856d7-6gs6q 0b1df59207deb6abc0fe0a196d38ab6f516e7b3440843892dbbbcb12646924dd network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-7c6df856d7-6gs6q 0b1df59207deb6abc0fe0a196d38ab6f516e7b3440843892dbbbcb12646924dd network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:8f:99:6a:8e:7e\",\"name\":\"0b1df59207deb6a\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5393ec87-25a3-4a74-8da4-4aec29fb6e23\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:45:51.266216 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:45:52.264680 3878 cni.go:403] [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default] I0326 23:45:52.357423 3878 cni.go:424] [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-95pjt-wbk5x 8dea2e0cb766ccc2b316a805ab7f2ee900e2c1950949d9cca2ed3505e21e65e6 network default NAD default], result "{}", err I0326 23:45:52.844819 3878 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:52.844853 3878 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-95pjt-wbk5x I0326 23:45:52.846420 3878 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-95pjt-wbk5x, ips: 10.129.0.42 I0326 23:45:52.846458 3878 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:45:53.268666 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:45:53.272556 3878 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:53.272598 3878 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-95pjt-wbk5x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:45:56.645781 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0326 23:45:58.301569 3878 cni.go:403] [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default] I0326 23:45:58.395923 3878 cni.go:424] [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines 6e22bfe60c5ff03afa7c05dd80cd3ce0accf378bd21867c42f5b4dbcc7c2f7ac network default NAD default], result "{}", err I0326 23:45:58.447232 3878 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:58.447273 3878 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0326 23:45:58.448790 3878 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.129.0.47 I0326 23:45:58.448820 3878 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:45:58.466830 3878 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.466857 3878 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:45:58.473596 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0326 23:45:58.473628 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0326 23:45:59.068081 3878 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-2-151.ec2.internal I0326 23:45:59.068175 3878 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0326 23:45:59.081344 3878 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 13.187849ms I0326 23:45:59.081380 3878 base_network_controller_pods.go:937] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 13.231322ms I0326 23:45:59.083028 3878 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 14.961494ms, libovsdb time 1.073157ms I0326 23:45:59.432288 3878 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default] I0326 23:45:59.441364 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 2df56a4b71f9a19, network: default, NAD default, SandboxID: "2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97", PCI device ID: , UID: "ae6e57eb-aa60-40cf-bf42-c1b92f323b2a", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0326 23:45:59.491973 3878 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:c7:1e:84:43:ca\",\"name\":\"2df56a4b71f9a19\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8637876-e57e-4120-a575-dfd81de598ec\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:02.326514 3878 cni.go:403] [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default] I0326 23:46:02.411844 3878 cni.go:424] [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 2df56a4b71f9a192d833b403be5660e2494fb1f6fc4ebf039c242273d3defd97 network default NAD default], result "{}", err I0326 23:46:02.465789 3878 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:02.465831 3878 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0326 23:46:02.467564 3878 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.129.0.48 I0326 23:46:02.467599 3878 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:02.488164 3878 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:02.488190 3878 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:02.495792 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0326 23:46:02.495894 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0326 23:46:02.710700 3878 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:02.710816 3878 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0326 23:46:02.724270 3878 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.483018ms I0326 23:46:02.724294 3878 base_network_controller_pods.go:937] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.515608ms I0326 23:46:02.725739 3878 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 15.058597ms, libovsdb time 1.050317ms I0326 23:46:03.074660 3878 cni.go:403] [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default] ADD starting CNI request [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default] I0326 23:46:03.084654 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: 122475d9afce001, network: default, NAD default, SandboxID: "122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99", PCI device ID: , UID: "573809ad-f4c1-4c14-9833-eb423efc71c4", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0326 23:46:03.133651 3878 cni.go:424] [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default] ADD finished CNI request [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:1c:b8:52:f8:03\",\"name\":\"122475d9afce001\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/76373808-d246-4729-84be-e33af5bcbcf0\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:46:03.297864 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0326 23:46:03.743629 3878 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-lqmnx] creating logical port tsf-tpa_create-db-lqmnx for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:03.743741 3878 kube.go:256] Updating pod tsf-tpa/create-db-lqmnx I0326 23:46:03.756592 3878 pod.go:62] [tsf-tpa/create-db-lqmnx] pod update took 12.87537ms I0326 23:46:03.756643 3878 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-lqmnx] addLogicalPort annotation time took 12.932835ms I0326 23:46:03.757995 3878 pods.go:271] [tsf-tpa/create-db-lqmnx] addLogicalPort took 14.396786ms, libovsdb time 900.833µs I0326 23:46:04.167496 3878 cni.go:403] [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default] ADD starting CNI request [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default] I0326 23:46:04.176107 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-lqmnx, hostIfaceName: 42528ac78aa0d40, network: default, NAD default, SandboxID: "42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c", PCI device ID: , UID: "33234d6e-033d-4f3d-a3d8-9d572cad1349", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0326 23:46:04.225762 3878 cni.go:424] [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default] ADD finished CNI request [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:5f:11:da:13:c8\",\"name\":\"42528ac78aa0d40\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33918f82-64ae-4513-a7d1-b8db6ca9418a\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:46:04.277917 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:46:08.371037 3878 cni.go:403] [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default] DEL starting CNI request [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default] I0326 23:46:08.442305 3878 cni.go:424] [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default] DEL finished CNI request [tsf-tpa/create-db-lqmnx 42528ac78aa0d409b17398b36ce074aac82657e00209046626ab47679426373c network default NAD default], result "{}", err I0326 23:46:08.494852 3878 obj_retry.go:464] Detected object tsf-tpa/create-db-lqmnx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:08.494893 3878 pods.go:176] Deleting pod: tsf-tpa/create-db-lqmnx I0326 23:46:08.496411 3878 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-lqmnx, ips: 10.129.0.50 I0326 23:46:08.496444 3878 obj_retry.go:464] Detected object tsf-tpa/create-db-lqmnx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:08.686436 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0326 23:46:09.519509 3878 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:09.519539 3878 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:09.619834 3878 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-5nwz9] creating logical port tsf-tpa_migrate-db-5nwz9 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:09.619971 3878 kube.go:256] Updating pod tsf-tpa/migrate-db-5nwz9 I0326 23:46:09.633872 3878 pod.go:62] [tsf-tpa/migrate-db-5nwz9] pod update took 13.925436ms I0326 23:46:09.633897 3878 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-5nwz9] addLogicalPort annotation time took 13.957101ms I0326 23:46:09.635181 3878 pods.go:271] [tsf-tpa/migrate-db-5nwz9] addLogicalPort took 15.364879ms, libovsdb time 851.713µs I0326 23:46:09.981759 3878 cni.go:403] [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default] I0326 23:46:09.990458 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-5nwz9, hostIfaceName: 1c1339834038819, network: default, NAD default, SandboxID: "1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f", PCI device ID: , UID: "3e70e593-3d02-4e0d-951e-b11f6e7fe1be", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0326 23:46:10.039013 3878 cni.go:424] [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f7:64:65:af:88\",\"name\":\"1c1339834038819\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/109d30bd-5dd6-4b16-8465-ad51a410049c\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:11.507719 3878 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-pcfsj-lpqtx] creating logical port tsf-tas_tuf-repository-init-pcfsj-lpqtx for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:11.507872 3878 kube.go:256] Updating pod tsf-tas/tuf-repository-init-pcfsj-lpqtx I0326 23:46:11.520223 3878 pod.go:62] [tsf-tas/tuf-repository-init-pcfsj-lpqtx] pod update took 12.384988ms I0326 23:46:11.520251 3878 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-pcfsj-lpqtx] addLogicalPort annotation time took 12.419086ms I0326 23:46:11.521809 3878 pods.go:271] [tsf-tas/tuf-repository-init-pcfsj-lpqtx] addLogicalPort took 14.112607ms, libovsdb time 1.117891ms I0326 23:46:11.653323 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0326 23:46:12.394521 3878 cni.go:403] [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default] I0326 23:46:12.487096 3878 cni.go:424] [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-5nwz9 1c1339834038819e0df8e318f98b7be16ad5141510c27d6739f7639b0e9ace4f network default NAD default], result "{}", err I0326 23:46:12.536991 3878 obj_retry.go:464] Detected object tsf-tpa/migrate-db-5nwz9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:12.537033 3878 pods.go:176] Deleting pod: tsf-tpa/migrate-db-5nwz9 I0326 23:46:12.538361 3878 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-5nwz9, ips: 10.129.0.51 I0326 23:46:12.538412 3878 obj_retry.go:464] Detected object tsf-tpa/migrate-db-5nwz9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:12.655088 3878 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:12.655122 3878 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:12.828724 3878 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-stcj7] creating logical port tsf-tpa_create-importers-stcj7 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:12.828820 3878 kube.go:256] Updating pod tsf-tpa/create-importers-stcj7 I0326 23:46:12.842779 3878 pod.go:62] [tsf-tpa/create-importers-stcj7] pod update took 13.984297ms I0326 23:46:12.842809 3878 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-stcj7] addLogicalPort annotation time took 14.021167ms I0326 23:46:12.844124 3878 pods.go:271] [tsf-tpa/create-importers-stcj7] addLogicalPort took 15.419266ms, libovsdb time 904.435µs I0326 23:46:13.192776 3878 cni.go:403] [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default] I0326 23:46:13.201740 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-stcj7, hostIfaceName: ea2c6a5f38f2a28, network: default, NAD default, SandboxID: "ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2", PCI device ID: , UID: "5c22d26e-260d-4b73-98c0-6c96b90dabba", MAC: 0a:58:0a:81:00:35, IPs: [10.129.0.53/23] I0326 23:46:13.249732 3878 cni.go:424] [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:d2:85:09:a9:60\",\"name\":\"ea2c6a5f38f2a28\"},{\"mac\":\"0a:58:0a:81:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d34ff7c8-5bb8-4740-9664-efecaaad43cc\"}],\"ips\":[{\"address\":\"10.129.0.53/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:15.417730 3878 cni.go:403] [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default] I0326 23:46:15.494809 3878 cni.go:424] [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-stcj7 ea2c6a5f38f2a280d355706644bfab1cec34e7962d21828d2ff9d75af6e8e3d2 network default NAD default], result "{}", err I0326 23:46:15.563409 3878 obj_retry.go:464] Detected object tsf-tpa/create-importers-stcj7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:15.563449 3878 pods.go:176] Deleting pod: tsf-tpa/create-importers-stcj7 I0326 23:46:15.564966 3878 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-stcj7, ips: 10.129.0.53 I0326 23:46:15.564992 3878 obj_retry.go:464] Detected object tsf-tpa/create-importers-stcj7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:15.864181 3878 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:15.864202 3878 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:46:28.411653 3878 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0326 23:46:31.506431 3878 cni.go:403] [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default] DEL starting CNI request [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default] I0326 23:46:31.581080 3878 cni.go:424] [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default] DEL finished CNI request [tsf-tpa/test-tsf-tpa 122475d9afce0012e4aab7f8cc16bc886c51d0dd314076630e76a28559cebb99 network default NAD default], result "{}", err I0326 23:46:31.633131 3878 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:31.633162 3878 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0326 23:46:31.634620 3878 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.129.0.49 I0326 23:46:31.634655 3878 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:31.655956 3878 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:31.655979 3878 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:31.663695 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0326 23:46:31.663720 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0326 23:46:32.418825 3878 namespace.go:98] [integration-service] adding namespace I0326 23:46:32.419950 3878 namespace.go:102] [integration-service] adding namespace took 1.103137ms I0326 23:46:32.428126 3878 namespace.go:98] [build-service] adding namespace I0326 23:46:32.428860 3878 namespace.go:102] [build-service] adding namespace took 715.109µs I0326 23:46:32.428899 3878 namespace.go:98] [image-controller] adding namespace I0326 23:46:32.429414 3878 namespace.go:102] [image-controller] adding namespace took 505.727µs I0326 23:46:32.499207 3878 namespace.go:142] [integration-service] updating namespace I0326 23:46:32.543545 3878 namespace.go:142] [integration-service] updating namespace I0326 23:46:32.564284 3878 namespace.go:142] [build-service] updating namespace I0326 23:46:32.599014 3878 namespace.go:142] [build-service] updating namespace I0326 23:46:32.608942 3878 namespace.go:142] [build-service] updating namespace I0326 23:46:32.624101 3878 namespace.go:142] [image-controller] updating namespace I0326 23:46:32.624145 3878 namespace.go:142] [integration-service] updating namespace I0326 23:46:32.645691 3878 namespace.go:142] [image-controller] updating namespace I0326 23:46:32.657910 3878 namespace.go:98] [release-service] adding namespace I0326 23:46:32.658823 3878 namespace.go:102] [release-service] adding namespace took 893.369µs I0326 23:46:32.684055 3878 namespace.go:142] [konflux-ui] updating namespace I0326 23:46:32.684205 3878 namespace.go:142] [release-service] updating namespace I0326 23:46:32.706657 3878 namespace.go:142] [release-service] updating namespace I0326 23:46:32.725594 3878 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-94t2j] creating logical port konflux-ui_tsf-konflux-public-key-94t2j for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:32.725691 3878 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-94t2j I0326 23:46:32.746938 3878 pod.go:62] [konflux-ui/tsf-konflux-public-key-94t2j] pod update took 21.270945ms I0326 23:46:32.746966 3878 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-94t2j] addLogicalPort annotation time took 21.301995ms I0326 23:46:32.748282 3878 pods.go:271] [konflux-ui/tsf-konflux-public-key-94t2j] addLogicalPort took 22.705023ms, libovsdb time 888.968µs I0326 23:46:32.785495 3878 namespace.go:98] [konflux-info] adding namespace I0326 23:46:32.786337 3878 namespace.go:102] [konflux-info] adding namespace took 822.163µs I0326 23:46:32.855751 3878 namespace.go:142] [konflux-info] updating namespace I0326 23:46:32.855868 3878 namespace.go:98] [namespace-lister] adding namespace I0326 23:46:32.856721 3878 namespace.go:102] [namespace-lister] adding namespace took 836.43µs I0326 23:46:32.919042 3878 namespace.go:98] [enterprise-contract-service] adding namespace I0326 23:46:32.920011 3878 namespace.go:102] [enterprise-contract-service] adding namespace took 947.376µs I0326 23:46:32.954024 3878 namespace.go:142] [image-controller] updating namespace I0326 23:46:32.992268 3878 namespace.go:142] [konflux-info] updating namespace I0326 23:46:33.071097 3878 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:33.162223 3878 namespace.go:98] [default-tenant] adding namespace I0326 23:46:33.163064 3878 namespace.go:102] [default-tenant] adding namespace took 815.343µs I0326 23:46:33.217214 3878 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:33.296172 3878 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:33.339541 3878 namespace.go:98] [segment-bridge] adding namespace I0326 23:46:33.340381 3878 namespace.go:102] [segment-bridge] adding namespace took 818.333µs I0326 23:46:33.386218 3878 namespace.go:142] [default-tenant] updating namespace I0326 23:46:33.386541 3878 cni.go:403] [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default] I0326 23:46:33.395683 3878 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-94t2j, hostIfaceName: eb88e3abf36d66e, network: default, NAD default, SandboxID: "eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6", PCI device ID: , UID: "401f06b3-1ff8-454f-be4d-17fb2e4d7bec", MAC: 0a:58:0a:81:00:36, IPs: [10.129.0.54/23] I0326 23:46:33.445992 3878 cni.go:424] [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:4a:71:fa:67:36\",\"name\":\"eb88e3abf36d66e\"},{\"mac\":\"0a:58:0a:81:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/59c78a54-6207-4c4e-8590-6c61165c781f\"}],\"ips\":[{\"address\":\"10.129.0.54/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:33.480125 3878 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:33.622347 3878 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:33.622489 3878 namespace.go:142] [default-tenant] updating namespace I0326 23:46:33.780564 3878 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:34.152873 3878 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0326 23:46:34.153508 3878 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0326 23:46:34.153526 3878 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0326 23:46:34.156241 3878 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0326 23:46:34.202187 3878 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0326 23:46:34.202214 3878 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0326 23:46:34.203104 3878 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0326 23:46:34.226373 3878 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-lnqzp] creating logical port namespace-lister_namespace-lister-67655c4674-lnqzp for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:34.226478 3878 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-lnqzp I0326 23:46:34.272448 3878 pod.go:62] [namespace-lister/namespace-lister-67655c4674-lnqzp] pod update took 45.995803ms I0326 23:46:34.272471 3878 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-lnqzp] addLogicalPort annotation time took 46.024593ms I0326 23:46:34.273760 3878 pods.go:271] [namespace-lister/namespace-lister-67655c4674-lnqzp] addLogicalPort took 47.407644ms, libovsdb time 869.291µs I0326 23:46:34.708062 3878 namespace.go:142] [build-service] updating namespace I0326 23:46:34.792151 3878 base_network_controller_pods.go:487] [default/konflux-ui/dex-78fc959b5b-qvd4g] creating logical port konflux-ui_dex-78fc959b5b-qvd4g for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:34.792246 3878 kube.go:256] Updating pod konflux-ui/dex-78fc959b5b-qvd4g I0326 23:46:34.827146 3878 pod.go:62] [konflux-ui/dex-78fc959b5b-qvd4g] pod update took 34.925599ms I0326 23:46:34.827173 3878 base_network_controller_pods.go:937] [default/konflux-ui/dex-78fc959b5b-qvd4g] addLogicalPort annotation time took 34.958172ms I0326 23:46:34.828433 3878 pods.go:271] [konflux-ui/dex-78fc959b5b-qvd4g] addLogicalPort took 36.297359ms, libovsdb time 806.191µs I0326 23:46:35.095718 3878 namespace.go:142] [default-tenant] updating namespace I0326 23:46:35.175339 3878 cni.go:403] [namespace-lister/namespace-lister-67655c4674-lnqzp 2a9c72d2ea5767a242baa062d7a2aede1e68d901b74b5eeda96da85015fcfc28 network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-lnqzp 2a9c72d2ea5767a242baa062d7a2aede1e68d901b74b5eeda96da85015fcfc28 network default NAD default] I0326 23:46:35.177348 3878 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-h2lcc] creating logical port integration-service_integration-service-controller-manager-759977d69f-h2lcc for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:35.177486 3878 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-h2lcc I0326 23:46:35.185176 3878 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-lnqzp, hostIfaceName: 2a9c72d2ea5767a, network: default, NAD default, SandboxID: "2a9c72d2ea5767a242baa062d7a2aede1e68d901b74b5eeda96da85015fcfc28", PCI device ID: , UID: "46d5be18-e5ed-4747-8e5d-9f57c7d499cb", MAC: 0a:58:0a:81:00:37, IPs: [10.129.0.55/23] I0326 23:46:35.193621 3878 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-h2lcc] pod update took 16.177454ms I0326 23:46:35.193641 3878 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-h2lcc] addLogicalPort annotation time took 16.206303ms I0326 23:46:35.194917 3878 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-h2lcc] addLogicalPort took 17.589867ms, libovsdb time 866.305µs I0326 23:46:35.402798 3878 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:35.442152 3878 cni.go:424] [namespace-lister/namespace-lister-67655c4674-lnqzp 2a9c72d2ea5767a242baa062d7a2aede1e68d901b74b5eeda96da85015fcfc28 network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-lnqzp 2a9c72d2ea5767a242baa062d7a2aede1e68d901b74b5eeda96da85015fcfc28 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:e7:da:45:8c:48\",\"name\":\"2a9c72d2ea5767a\"},{\"mac\":\"0a:58:0a:81:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9bafe9d5-5151-40fe-863e-5cd8810a2e9f\"}],\"ips\":[{\"address\":\"10.129.0.55/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:35.673929 3878 namespace.go:142] [image-controller] updating namespace I0326 23:46:35.868203 3878 namespace.go:142] [integration-service] updating namespace I0326 23:46:35.961132 3878 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-x8w5g] creating logical port release-service_release-service-controller-manager-854f775895-x8w5g for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:35.961248 3878 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-x8w5g I0326 23:46:35.975144 3878 base_network_controller_pods.go:487] [default/konflux-ui/proxy-7c855bfc89-9pqrl] creating logical port konflux-ui_proxy-7c855bfc89-9pqrl for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:35.975247 3878 kube.go:256] Updating pod konflux-ui/proxy-7c855bfc89-9pqrl I0326 23:46:35.989022 3878 pod.go:62] [release-service/release-service-controller-manager-854f775895-x8w5g] pod update took 27.807322ms I0326 23:46:35.989045 3878 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-x8w5g] addLogicalPort annotation time took 27.83648ms I0326 23:46:35.990321 3878 pods.go:271] [release-service/release-service-controller-manager-854f775895-x8w5g] addLogicalPort took 29.212132ms, libovsdb time 879.235µs I0326 23:46:36.007652 3878 pod.go:62] [konflux-ui/proxy-7c855bfc89-9pqrl] pod update took 32.438565ms I0326 23:46:36.007689 3878 base_network_controller_pods.go:937] [default/konflux-ui/proxy-7c855bfc89-9pqrl] addLogicalPort annotation time took 32.478934ms I0326 23:46:36.009092 3878 pods.go:271] [konflux-ui/proxy-7c855bfc89-9pqrl] addLogicalPort took 33.968952ms, libovsdb time 852.829µs I0326 23:46:36.149848 3878 namespace.go:142] [konflux-info] updating namespace I0326 23:46:36.382911 3878 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:36.545904 3878 cni.go:403] [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default] I0326 23:46:36.597176 3878 namespace.go:142] [release-service] updating namespace I0326 23:46:36.624809 3878 cni.go:424] [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-94t2j eb88e3abf36d66eb4a51bb88b0d52ab7a2e2b7fb35148197eb6f222e8dcfa2c6 network default NAD default], result "{}", err I0326 23:46:36.688847 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:46:36.777557 3878 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:36.885201 3878 namespace.go:142] [build-service] updating namespace I0326 23:46:36.898160 3878 namespace.go:142] [default-tenant] updating namespace I0326 23:46:36.920182 3878 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:36.930116 3878 namespace.go:142] [image-controller] updating namespace I0326 23:46:36.947797 3878 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:36.947835 3878 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-94t2j I0326 23:46:36.949430 3878 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-94t2j, ips: 10.129.0.54 I0326 23:46:36.949464 3878 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:36.954199 3878 namespace.go:142] [integration-service] updating namespace I0326 23:46:36.977341 3878 namespace.go:142] [konflux-info] updating namespace I0326 23:46:36.999542 3878 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:37.012796 3878 namespace.go:142] [release-service] updating namespace I0326 23:46:37.035341 3878 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:37.080276 3878 namespace.go:142] [build-service] updating namespace I0326 23:46:37.126496 3878 namespace.go:142] [default-tenant] updating namespace I0326 23:46:37.208442 3878 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:46:37.306485 3878 namespace.go:142] [image-controller] updating namespace I0326 23:46:37.405215 3878 namespace.go:142] [integration-service] updating namespace I0326 23:46:37.505793 3878 namespace.go:142] [konflux-info] updating namespace W0326 23:46:37.554245 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0326 23:46:37.606112 3878 namespace.go:142] [namespace-lister] updating namespace I0326 23:46:37.653385 3878 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:37.653520 3878 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-94t2j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:37.704584 3878 namespace.go:142] [release-service] updating namespace I0326 23:46:37.805415 3878 namespace.go:142] [segment-bridge] updating namespace I0326 23:46:37.828863 3878 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:37.828957 3878 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0326 23:46:37.841741 3878 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 12.806294ms I0326 23:46:37.841771 3878 base_network_controller_pods.go:937] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 12.842839ms I0326 23:46:37.843222 3878 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 14.377677ms, libovsdb time 975.023µs W0326 23:46:37.888826 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0326 23:46:38.190781 3878 cni.go:403] [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default] ADD starting CNI request [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default] I0326 23:46:38.200025 3878 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: 36e40dd54d57997, network: default, NAD default, SandboxID: "36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9", PCI device ID: , UID: "b4786a68-c84c-4408-85b8-f7efdafa3c77", MAC: 0a:58:0a:81:00:3c, IPs: [10.129.0.60/23] I0326 23:46:38.248102 3878 cni.go:424] [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default] ADD finished CNI request [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:3c:4e:76:99:fd\",\"name\":\"36e40dd54d57997\"},{\"mac\":\"0a:58:0a:81:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e176e899-3562-4149-b332-bc8debe57173\"}],\"ips\":[{\"address\":\"10.129.0.60/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:46:38.560181 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister W0326 23:46:39.897518 3878 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0326 23:46:42.434971 3878 cni.go:403] [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default] I0326 23:46:42.445189 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-pcfsj-lpqtx, hostIfaceName: 922ff5ffab32d39, network: default, NAD default, SandboxID: "922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1", PCI device ID: , UID: "fc8431e5-435d-4f80-9212-648511a2d221", MAC: 0a:58:0a:81:00:34, IPs: [10.129.0.52/23] I0326 23:46:42.500634 3878 cni.go:424] [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:7d:d3:09:35:3f\",\"name\":\"922ff5ffab32d39\"},{\"mac\":\"0a:58:0a:81:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/30af7f50-d7cc-4a53-b156-be6d5807f034\"}],\"ips\":[{\"address\":\"10.129.0.52/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:42.665175 3878 cni.go:403] [konflux-ui/dex-78fc959b5b-qvd4g 394d3a67ff5b947ea64c7369a3cac56a879335bd00cf33d7871a20cc70bd8151 network default NAD default] ADD starting CNI request [konflux-ui/dex-78fc959b5b-qvd4g 394d3a67ff5b947ea64c7369a3cac56a879335bd00cf33d7871a20cc70bd8151 network default NAD default] I0326 23:46:42.674537 3878 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-78fc959b5b-qvd4g, hostIfaceName: 394d3a67ff5b947, network: default, NAD default, SandboxID: "394d3a67ff5b947ea64c7369a3cac56a879335bd00cf33d7871a20cc70bd8151", PCI device ID: , UID: "8940fad3-5b99-43d9-8d42-caa3dd7a2e6d", MAC: 0a:58:0a:81:00:38, IPs: [10.129.0.56/23] I0326 23:46:42.725196 3878 cni.go:424] [konflux-ui/dex-78fc959b5b-qvd4g 394d3a67ff5b947ea64c7369a3cac56a879335bd00cf33d7871a20cc70bd8151 network default NAD default] ADD finished CNI request [konflux-ui/dex-78fc959b5b-qvd4g 394d3a67ff5b947ea64c7369a3cac56a879335bd00cf33d7871a20cc70bd8151 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:8d:1f:df:16:4d\",\"name\":\"394d3a67ff5b947\"},{\"mac\":\"0a:58:0a:81:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/61600afa-2216-40b8-a36b-ead897304f3b\"}],\"ips\":[{\"address\":\"10.129.0.56/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:45.490667 3878 cni.go:403] [konflux-ui/proxy-7c855bfc89-9pqrl b2d38fcc6660456dddf1ea0c90dec914f3b8167c8a4a81d1c1d27465ee3cc990 network default NAD default] ADD starting CNI request [konflux-ui/proxy-7c855bfc89-9pqrl b2d38fcc6660456dddf1ea0c90dec914f3b8167c8a4a81d1c1d27465ee3cc990 network default NAD default] I0326 23:46:45.490721 3878 cni.go:403] [release-service/release-service-controller-manager-854f775895-x8w5g bba57d5294b54c448b2a6af0ffd1e0c6637930788fd1c391fc4f63b5a9aed6dd network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-x8w5g bba57d5294b54c448b2a6af0ffd1e0c6637930788fd1c391fc4f63b5a9aed6dd network default NAD default] I0326 23:46:45.490988 3878 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default] I0326 23:46:45.502528 3878 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-7c855bfc89-9pqrl, hostIfaceName: b2d38fcc6660456, network: default, NAD default, SandboxID: "b2d38fcc6660456dddf1ea0c90dec914f3b8167c8a4a81d1c1d27465ee3cc990", PCI device ID: , UID: "3168ccdc-cbff-4085-b304-4618b5b06257", MAC: 0a:58:0a:81:00:3b, IPs: [10.129.0.59/23] I0326 23:46:45.503971 3878 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-h2lcc, hostIfaceName: ec8589aaa348a1a, network: default, NAD default, SandboxID: "ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c", PCI device ID: , UID: "d1568f7b-1f5c-4f7c-ba63-e6d898127929", MAC: 0a:58:0a:81:00:39, IPs: [10.129.0.57/23] I0326 23:46:45.504626 3878 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-x8w5g, hostIfaceName: bba57d5294b54c4, network: default, NAD default, SandboxID: "bba57d5294b54c448b2a6af0ffd1e0c6637930788fd1c391fc4f63b5a9aed6dd", PCI device ID: , UID: "d5d572f5-f7df-4b1c-b8da-dc1c2da3acb9", MAC: 0a:58:0a:81:00:3a, IPs: [10.129.0.58/23] I0326 23:46:45.561081 3878 cni.go:424] [release-service/release-service-controller-manager-854f775895-x8w5g bba57d5294b54c448b2a6af0ffd1e0c6637930788fd1c391fc4f63b5a9aed6dd network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-x8w5g bba57d5294b54c448b2a6af0ffd1e0c6637930788fd1c391fc4f63b5a9aed6dd network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:75:68:75:3c:e5\",\"name\":\"bba57d5294b54c4\"},{\"mac\":\"0a:58:0a:81:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7313eeed-3ff3-4a6e-a5aa-78ac1777c089\"}],\"ips\":[{\"address\":\"10.129.0.58/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:45.787510 3878 cni.go:424] [konflux-ui/proxy-7c855bfc89-9pqrl b2d38fcc6660456dddf1ea0c90dec914f3b8167c8a4a81d1c1d27465ee3cc990 network default NAD default] ADD finished CNI request [konflux-ui/proxy-7c855bfc89-9pqrl b2d38fcc6660456dddf1ea0c90dec914f3b8167c8a4a81d1c1d27465ee3cc990 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:a8:f1:1a:bf:0f\",\"name\":\"b2d38fcc6660456\"},{\"mac\":\"0a:58:0a:81:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92a3d096-dc18-42bf-8056-e7e0c21bca74\"}],\"ips\":[{\"address\":\"10.129.0.59/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:45.788241 3878 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:8a:19:db:c5:fb\",\"name\":\"ec8589aaa348a1a\"},{\"mac\":\"0a:58:0a:81:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2dc43102-830f-46ab-a3a3-a6821e5f4a2f\"}],\"ips\":[{\"address\":\"10.129.0.57/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:46:47.648929 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:46:48.652112 3878 cni.go:403] [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default] I0326 23:46:48.736890 3878 cni.go:424] [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-pcfsj-lpqtx 922ff5ffab32d392dbc832a0602dc1846ac3a6180f53311560792f4ff42e90a1 network default NAD default], result "{}", err W0326 23:46:49.963720 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:46:49.966133 3878 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:46:49.991291 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:46:49.991316 3878 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0326 23:46:50.957219 3878 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:50.957252 3878 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-pcfsj-lpqtx I0326 23:46:50.958849 3878 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-pcfsj-lpqtx, ips: 10.129.0.52 I0326 23:46:50.958883 3878 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:46:51.981679 3878 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:46:51.981709 3878 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pcfsj-lpqtx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:46:52.673871 3878 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:46:52.675264 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:46:52.700282 3878 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:46:52.700301 3878 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:46:52.728245 3878 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0326 23:46:52.754018 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:46:52.755379 3878 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0326 23:46:54.026434 3878 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-496g7] creating logical port tsf-tas_tuf-bc5768c88-496g7 for pod on switch ip-10-0-2-151.ec2.internal I0326 23:46:54.026535 3878 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-496g7 I0326 23:46:54.044189 3878 pod.go:62] [tsf-tas/tuf-bc5768c88-496g7] pod update took 17.682154ms I0326 23:46:54.044212 3878 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-496g7] addLogicalPort annotation time took 17.713031ms I0326 23:46:54.045526 3878 pods.go:271] [tsf-tas/tuf-bc5768c88-496g7] addLogicalPort took 19.138983ms, libovsdb time 875.288µs I0326 23:47:00.689421 3878 cni.go:403] [tsf-tas/tuf-bc5768c88-496g7 61c7ec94c7a96fdf0a8c94afa36ecacb60b09770ce24ee9e85f55c8ba44c19bc network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-496g7 61c7ec94c7a96fdf0a8c94afa36ecacb60b09770ce24ee9e85f55c8ba44c19bc network default NAD default] I0326 23:47:00.698760 3878 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-496g7, hostIfaceName: 61c7ec94c7a96fd, network: default, NAD default, SandboxID: "61c7ec94c7a96fdf0a8c94afa36ecacb60b09770ce24ee9e85f55c8ba44c19bc", PCI device ID: , UID: "79aa7d73-622c-4b79-b127-5436ec5da223", MAC: 0a:58:0a:81:00:3d, IPs: [10.129.0.61/23] I0326 23:47:00.750684 3878 cni.go:424] [tsf-tas/tuf-bc5768c88-496g7 61c7ec94c7a96fdf0a8c94afa36ecacb60b09770ce24ee9e85f55c8ba44c19bc network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-496g7 61c7ec94c7a96fdf0a8c94afa36ecacb60b09770ce24ee9e85f55c8ba44c19bc network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:04:6a:5f:0c:35\",\"name\":\"61c7ec94c7a96fd\"},{\"mac\":\"0a:58:0a:81:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96c229a3-b43d-4ee1-9159-8c281cc848be\"}],\"ips\":[{\"address\":\"10.129.0.61/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:47:00.980924 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:47:00.980959 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:47:00.981061 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:47:00.981088 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0326 23:47:01.170245 3878 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default] I0326 23:47:01.251571 3878 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-h2lcc ec8589aaa348a1a59e1a6066f6ed650e05c28f3ad2009d9166ce378178d1d03c network default NAD default], result "{}", err I0326 23:47:01.292799 3878 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:01.292842 3878 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-h2lcc I0326 23:47:01.294240 3878 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-h2lcc, ips: 10.129.0.57 I0326 23:47:01.294269 3878 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:01.737283 3878 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:01.737320 3878 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-h2lcc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:01.743562 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-h2lcc I0326 23:47:01.743590 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-h2lcc W0326 23:47:02.748671 3878 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0326 23:47:03.685788 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:47:03.689626 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:47:05.700789 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex I0326 23:47:12.562976 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=67 W0326 23:47:13.760036 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf W0326 23:47:28.036330 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:28.036357 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:28.036371 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:28.036376 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:28.733538 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0326 23:47:28.733562 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0326 23:47:42.665276 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0326 23:47:58.353242 3878 obj_retry.go:464] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:58.353884 3878 obj_retry.go:464] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:47:58.682020 3878 obj_retry.go:458] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:58.682063 3878 obj_retry.go:458] Detected object konflux-ui/proxy-7b5fc4779b-vk52h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:47:58.689623 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-7b5fc4779b-vk52h I0326 23:47:58.689651 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-7b5fc4779b-vk52h I0326 23:48:08.144720 3878 cni.go:403] [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default] DEL starting CNI request [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default] I0326 23:48:08.227481 3878 cni.go:424] [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default] DEL finished CNI request [konflux-ui/test-tsf-konflux 36e40dd54d57997eb3439f9628e0a1142d88171112838f6c34a3dc177ddb93f9 network default NAD default], result "{}", err I0326 23:48:08.285662 3878 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:08.285694 3878 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0326 23:48:08.287025 3878 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.129.0.60 I0326 23:48:08.287055 3878 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:08.325970 3878 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:08.325998 3878 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:08.337480 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0326 23:48:08.337507 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0326 23:48:16.680986 3878 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0326 23:48:30.660509 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:48:33.815501 3878 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:33.816102 3878 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:34.022048 3878 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:34.022070 3878 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:34.027855 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 I0326 23:48:34.027869 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-98dz2 I0326 23:48:46.951712 3878 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:46.952324 3878 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:48:47.676663 3878 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:47.676687 3878 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:48:47.683313 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg I0326 23:48:47.683342 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-mbgxg I0326 23:50:04.292875 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:04.293523 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:05.159312 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.159337 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-v6wgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:13.675566 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0326 23:50:58.563327 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0326 23:52:04.691000 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0326 23:52:07.652239 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:52:47.565074 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=96 I0326 23:52:50.672584 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0326 23:52:51.569967 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=24 I0326 23:52:58.651277 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0326 23:53:04.571840 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=66 I0326 23:53:13.655923 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 W0326 23:53:26.967477 3878 duration.go:461] Config duration recorder: measurement expired for pod/openshift-image-registry/image-registry-5fd5865cf4-s6vd6 I0326 23:53:45.573385 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=246 I0326 23:53:51.565367 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=30 I0326 23:54:41.683951 3878 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:55:57.697881 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0326 23:56:09.049262 3878 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:09.049307 3878 obj_retry.go:458] Detected object tsf-tpa/create-db-lqmnx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:09.054427 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-lqmnx I0326 23:56:09.054463 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-lqmnx I0326 23:56:12.031429 3878 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:12.031490 3878 obj_retry.go:458] Detected object tsf-tpa/migrate-db-5nwz9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:12.035128 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-5nwz9 I0326 23:56:12.035160 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-5nwz9 I0326 23:56:15.032159 3878 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:15.032215 3878 obj_retry.go:458] Detected object tsf-tpa/create-importers-stcj7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:15.036750 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-stcj7 I0326 23:56:15.036777 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-stcj7 I0326 23:56:22.566788 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0326 23:56:22.668098 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0326 23:57:08.584658 3878 namespace.go:98] [default-tenant-managed] adding namespace I0326 23:57:08.585644 3878 namespace.go:102] [default-tenant-managed] adding namespace took 962.939µs I0326 23:57:08.612202 3878 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:08.627530 3878 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:09.669308 3878 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:09.802549 3878 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:09.901913 3878 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:57:28.654916 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0326 23:58:08.746053 3878 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-25hvj for pod on switch ip-10-0-2-151.ec2.internal I0326 23:58:08.746154 3878 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj I0326 23:58:08.761091 3878 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj] pod update took 14.965746ms I0326 23:58:08.761113 3878 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj] addLogicalPort annotation time took 14.992503ms I0326 23:58:08.762551 3878 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj] addLogicalPort took 16.521751ms, libovsdb time 978.659µs I0326 23:58:09.111582 3878 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default] I0326 23:58:09.120911 3878 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-25hvj, hostIfaceName: 89c24ff8a6a20cd, network: default, NAD default, SandboxID: "89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd", PCI device ID: , UID: "25aeda61-6f6a-41c9-b92e-ef1f4256bdee", MAC: 0a:58:0a:81:00:3e, IPs: [10.129.0.62/23] I0326 23:58:09.172054 3878 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:88:2d:2d:23:e8\",\"name\":\"89c24ff8a6a20cd\"},{\"mac\":\"0a:58:0a:81:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20c06625-e122-4b8c-a250-c9ed97f01da1\"}],\"ips\":[{\"address\":\"10.129.0.62/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:58:12.262356 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:12.262916 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:13.033031 3878 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-clone-repository-pod for pod on switch ip-10-0-2-151.ec2.internal I0326 23:58:13.033150 3878 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod I0326 23:58:13.053226 3878 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod] pod update took 20.111038ms I0326 23:58:13.053251 3878 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod] addLogicalPort annotation time took 20.140805ms I0326 23:58:13.054540 3878 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod] addLogicalPort took 21.524246ms, libovsdb time 857.301µs I0326 23:58:13.397462 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default] I0326 23:58:13.406490 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-clone-repository-pod, hostIfaceName: 0ecf2b846e38208, network: default, NAD default, SandboxID: "0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923", PCI device ID: , UID: "550caf07-23a4-4145-90d5-2cc5bb137239", MAC: 0a:58:0a:81:00:3f, IPs: [10.129.0.63/23] I0326 23:58:13.460506 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f3:67:a0:6f:7b\",\"name\":\"0ecf2b846e38208\"},{\"mac\":\"0a:58:0a:81:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bfb62834-480f-40d3-ba08-5badc09365fd\"}],\"ips\":[{\"address\":\"10.129.0.63/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0326 23:58:22.095437 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095464 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095473 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095478 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095492 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095498 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095502 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:22.095506 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:23.471726 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:23.471750 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:23.471755 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:58:23.471761 3878 util.go:959] Failed to build node endpoints for node ip-10-0-2-151.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:58:23.663474 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:58:28.678412 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0326 23:58:38.634685 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default] I0326 23:58:38.724284 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod 0ecf2b846e38208edc19c4757041bde8beabec020191d8cbd93be5ee9ee60923 network default NAD default], result "{}", err I0326 23:58:38.805917 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.805957 3878 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod I0326 23:58:38.807324 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod, ips: 10.129.0.63 I0326 23:58:38.807353 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.995089 3878 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod for pod on switch ip-10-0-2-151.ec2.internal I0326 23:58:38.995260 3878 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod I0326 23:58:39.024868 3878 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod] pod update took 29.657767ms I0326 23:58:39.024895 3878 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod] addLogicalPort annotation time took 29.691693ms I0326 23:58:39.026321 3878 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod] addLogicalPort took 31.25267ms, libovsdb time 1.010562ms I0326 23:58:39.361767 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default] I0326 23:58:39.370416 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod, hostIfaceName: 358c998b2fb340f, network: default, NAD default, SandboxID: "358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49", PCI device ID: , UID: "faa8c457-c058-4acc-b1f1-eded436131ec", MAC: 0a:58:0a:81:00:40, IPs: [10.129.0.64/23] I0326 23:58:39.422530 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:a5:4a:0f:33:13\",\"name\":\"358c998b2fb340f\"},{\"mac\":\"0a:58:0a:81:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/73ee7d41-b32e-4bb5-bff5-36d9b07ab372\"}],\"ips\":[{\"address\":\"10.129.0.64/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:58:57.781752 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default] I0326 23:58:57.868236 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod 358c998b2fb340f4739a64614b842415dea4aa257a266e608bf9e90d77e00c49 network default NAD default], result "{}", err I0326 23:58:57.960051 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:57.960098 3878 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod I0326 23:58:57.961515 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod, ips: 10.129.0.64 I0326 23:58:57.961543 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:59.612161 3878 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-build-container-pod for pod on switch ip-10-0-2-151.ec2.internal I0326 23:58:59.612294 3878 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod I0326 23:58:59.678939 3878 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod] pod update took 66.682818ms I0326 23:58:59.678971 3878 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod] addLogicalPort annotation time took 66.722987ms I0326 23:58:59.680366 3878 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod] addLogicalPort took 68.226123ms, libovsdb time 1.003113ms I0326 23:58:59.986009 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default] I0326 23:58:59.994683 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-build-container-pod, hostIfaceName: bbde69c8c7583d2, network: default, NAD default, SandboxID: "bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766", PCI device ID: , UID: "cf18f233-9c0b-4c39-b91d-e18aad46647f", MAC: 0a:58:0a:81:00:41, IPs: [10.129.0.65/23] I0326 23:59:00.047037 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:8f:02:2d:cd:41\",\"name\":\"bbde69c8c7583d2\"},{\"mac\":\"0a:58:0a:81:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c9863e9-7704-4270-a7a3-4e07f8c000c8\"}],\"ips\":[{\"address\":\"10.129.0.65/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0326 23:59:28.694691 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 00:00:02.964348 3878 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:02.964943 3878 obj_retry.go:464] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.849844 3878 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.850324 3878 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.977416 3878 obj_retry.go:464] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.978016 3878 obj_retry.go:464] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:03.988771 3878 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:03.988791 3878 obj_retry.go:458] Detected object openshift-image-registry/image-pruner-29576160-kbqb2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.869978 3878 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.870004 3878 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29576160-76b87 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.982118 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.982770 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:04.996804 3878 obj_retry.go:458] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:04.996837 3878 obj_retry.go:458] Detected object integration-service/integration-service-snapshot-garbage-collector-29576160-rtpc7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.003230 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.003262 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576160-n888d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.031127 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.031151 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:06.035518 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 I0327 00:00:06.035540 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-zs7r9 I0327 00:00:14.968480 3878 obj_retry.go:464] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:14.969079 3878 obj_retry.go:464] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:15.991197 3878 obj_retry.go:458] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:15.991220 3878 obj_retry.go:458] Detected object tsf-tas/backfill-redis-29576160-lx92g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:16.654365 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 00:00:18.575350 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0327 00:00:23.687062 3878 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0327 00:00:27.573239 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0327 00:00:54.337472 3878 obj_retry.go:464] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:54.338024 3878 obj_retry.go:464] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:00:55.195641 3878 obj_retry.go:458] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:55.195663 3878 obj_retry.go:458] Detected object tsf-tas/segment-backup-nightly-metrics-29576160-d284n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:00:58.567512 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=15 I0327 00:01:13.700362 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0327 00:01:29.658690 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0327 00:01:37.803579 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default] I0327 00:01:37.895139 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod bbde69c8c7583d2171041bf1b59f141e78039e1dd6b3955dfacede8000f40766 network default NAD default], result "{}", err I0327 00:01:38.013262 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:01:38.013302 3878 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod I0327 00:01:38.014923 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod, ips: 10.129.0.65 I0327 00:01:38.014951 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:01:48.568559 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0327 00:02:20.400961 3878 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod for pod on switch ip-10-0-2-151.ec2.internal I0327 00:02:20.401081 3878 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod I0327 00:02:20.423985 3878 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod] pod update took 22.928668ms I0327 00:02:20.424008 3878 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod] addLogicalPort annotation time took 22.956996ms I0327 00:02:20.425543 3878 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod] addLogicalPort took 24.598601ms, libovsdb time 1.079804ms I0327 00:02:20.438769 3878 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-clair-scan-pod for pod on switch ip-10-0-2-151.ec2.internal I0327 00:02:20.438998 3878 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod I0327 00:02:20.466548 3878 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod] pod update took 27.595494ms I0327 00:02:20.466580 3878 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod] addLogicalPort annotation time took 27.633355ms I0327 00:02:20.467964 3878 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod] addLogicalPort took 29.210993ms, libovsdb time 928.392µs I0327 00:02:20.767606 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default] I0327 00:02:20.777357 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod, hostIfaceName: 89993b9306f64d8, network: default, NAD default, SandboxID: "89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533", PCI device ID: , UID: "987fc2e5-b30b-43ea-8a4a-40f2f566b63b", MAC: 0a:58:0a:81:00:42, IPs: [10.129.0.66/23] I0327 00:02:20.809465 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default] I0327 00:02:20.821753 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-clair-scan-pod, hostIfaceName: 53731eea245674e, network: default, NAD default, SandboxID: "53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a", PCI device ID: , UID: "55bdda5c-6ed0-45d9-b588-c8e695063a8a", MAC: 0a:58:0a:81:00:43, IPs: [10.129.0.67/23] I0327 00:02:20.833776 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:c8:f8:27:1d:8c\",\"name\":\"89993b9306f64d8\"},{\"mac\":\"0a:58:0a:81:00:42\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b86e99fd-c614-490c-bd21-b2b185ee3a8a\"}],\"ips\":[{\"address\":\"10.129.0.66/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 00:02:20.873054 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:99:67:3e:d7:5b\",\"name\":\"53731eea245674e\"},{\"mac\":\"0a:58:0a:81:00:43\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ff33db7-a34c-430b-b94f-dbe2f0bbc765\"}],\"ips\":[{\"address\":\"10.129.0.67/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 00:02:21.067490 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:21.068118 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:02:21.432665 3878 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod] creating logical port default-tenant_tsf-comp-qthr-on-push-fh55d-clamav-scan-pod for pod on switch ip-10-0-2-151.ec2.internal I0327 00:02:21.432819 3878 kube.go:256] Updating pod default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod I0327 00:02:21.454049 3878 pod.go:62] [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod] pod update took 21.268801ms I0327 00:02:21.454076 3878 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod] addLogicalPort annotation time took 21.300359ms I0327 00:02:21.455408 3878 pods.go:271] [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod] addLogicalPort took 22.747525ms, libovsdb time 854.591µs I0327 00:02:21.675522 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 00:02:21.798822 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default] I0327 00:02:21.809064 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-qthr-on-push-fh55d-clamav-scan-pod, hostIfaceName: 25d91f6ada8407b, network: default, NAD default, SandboxID: "25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1", PCI device ID: , UID: "ca3f1080-8877-4366-ae24-ad7d7e29024c", MAC: 0a:58:0a:81:00:44, IPs: [10.129.0.68/23] I0327 00:02:21.861037 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:df:f5:b7:2d:3d\",\"name\":\"25d91f6ada8407b\"},{\"mac\":\"0a:58:0a:81:00:44\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8d6129a-7ac7-4bc5-b9e8-212c9118a230\"}],\"ips\":[{\"address\":\"10.129.0.68/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 00:03:04.711528 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:04.712292 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:05.461495 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:05.462276 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:19.674781 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:19.675516 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:22.576366 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=278 I0327 00:03:24.593198 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default] I0327 00:03:24.688635 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod 89993b9306f64d84638964be7b15db1ff8cdd3af38ab58774288cb8312f1d533 network default NAD default], result "{}", err I0327 00:03:24.798551 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:24.798592 3878 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod I0327 00:03:24.800165 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod, ips: 10.129.0.66 I0327 00:03:24.800198 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.634503 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default] I0327 00:03:28.724593 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod 25d91f6ada8407bc08501b915a00e7842072bd8d2d70169506ccfe1a600d2fb1 network default NAD default], result "{}", err I0327 00:03:28.744954 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.745606 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.822517 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.823194 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.861110 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.861714 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.894559 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:28.894598 3878 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod I0327 00:03:28.896149 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod, ips: 10.129.0.68 I0327 00:03:28.896185 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:53.013575 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:03:53.014228 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:12.115914 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:12.116488 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:04:25.569576 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0327 00:05:21.697290 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0327 00:05:39.496465 3878 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:39.496910 3878 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:39.645215 3878 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:39.645238 3878 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-jnwzv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:39.648521 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-multus/cni-sysctl-allowlist-ds-jnwzv I0327 00:05:39.648551 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-multus/cni-sysctl-allowlist-ds-jnwzv I0327 00:05:40.169273 3878 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.169776 3878 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.255292 3878 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.255314 3878 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-zs4tx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.260897 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-multus/cni-sysctl-allowlist-ds-zs4tx I0327 00:05:40.260917 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-multus/cni-sysctl-allowlist-ds-zs4tx I0327 00:05:40.420678 3878 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.420715 3878 obj_retry.go:464] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:40.544267 3878 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.544297 3878 obj_retry.go:458] Detected object openshift-multus/cni-sysctl-allowlist-ds-w5r9x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:05:40.551088 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-multus/cni-sysctl-allowlist-ds-w5r9x I0327 00:05:40.551108 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-multus/cni-sysctl-allowlist-ds-w5r9x I0327 00:05:46.595483 3878 cni.go:403] [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default] I0327 00:05:46.683380 3878 cni.go:424] [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod 53731eea245674e89a13f70e4a62567c05cadcf9a984eddccbcb093edb8cdf9a network default NAD default], result "{}", err I0327 00:05:46.766789 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:46.766828 3878 pods.go:176] Deleting pod: default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod I0327 00:05:46.768419 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod, ips: 10.129.0.67 I0327 00:05:46.768455 3878 obj_retry.go:464] Detected object default-tenant/tsf-comp-qthr-on-push-fh55d-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:05:55.658061 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0327 00:05:59.671360 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0327 00:06:12.495351 3878 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-xpec-klhcb-verify-pod] creating logical port default-tenant_my-integration-test-xpec-klhcb-verify-pod for pod on switch ip-10-0-2-151.ec2.internal I0327 00:06:12.495591 3878 kube.go:256] Updating pod default-tenant/my-integration-test-xpec-klhcb-verify-pod I0327 00:06:12.530024 3878 pod.go:62] [default-tenant/my-integration-test-xpec-klhcb-verify-pod] pod update took 34.478747ms I0327 00:06:12.530050 3878 base_network_controller_pods.go:937] [default/default-tenant/my-integration-test-xpec-klhcb-verify-pod] addLogicalPort annotation time took 34.510463ms I0327 00:06:12.531331 3878 pods.go:271] [default-tenant/my-integration-test-xpec-klhcb-verify-pod] addLogicalPort took 36.002543ms, libovsdb time 861.734µs I0327 00:06:12.864197 3878 cni.go:403] [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default] ADD starting CNI request [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default] I0327 00:06:12.873280 3878 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-xpec-klhcb-verify-pod, hostIfaceName: ed4b1fe34ae0d2b, network: default, NAD default, SandboxID: "ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de", PCI device ID: , UID: "f4e01473-152d-4b6a-9118-f6edf85c4892", MAC: 0a:58:0a:81:00:45, IPs: [10.129.0.69/23] I0327 00:06:12.925194 3878 cni.go:424] [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default] ADD finished CNI request [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:a7:cc:4d:19:d1\",\"name\":\"ed4b1fe34ae0d2b\"},{\"mac\":\"0a:58:0a:81:00:45\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5bbc58fe-b26f-4164-a1b3-aca1fc30eeaa\"}],\"ips\":[{\"address\":\"10.129.0.69/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0327 00:06:37.962467 3878 cni.go:403] [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default] DEL starting CNI request [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default] I0327 00:06:38.053423 3878 cni.go:424] [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default] DEL finished CNI request [default-tenant/my-integration-test-xpec-klhcb-verify-pod ed4b1fe34ae0d2b051d7c3c7bff43a4278e834b99ee03b72b7e280cb154024de network default NAD default], result "{}", err I0327 00:06:38.157935 3878 obj_retry.go:464] Detected object default-tenant/my-integration-test-xpec-klhcb-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:38.157971 3878 pods.go:176] Deleting pod: default-tenant/my-integration-test-xpec-klhcb-verify-pod I0327 00:06:38.159575 3878 pods.go:236] Attempting to release IPs for pod: default-tenant/my-integration-test-xpec-klhcb-verify-pod, ips: 10.129.0.69 I0327 00:06:38.159601 3878 obj_retry.go:464] Detected object default-tenant/my-integration-test-xpec-klhcb-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:06:38.680907 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 00:07:16.666225 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0327 00:07:30.589454 3878 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 00:07:31.658304 3878 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 00:07:41.703602 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0327 00:07:57.570649 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=7 I0327 00:08:00.576373 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0327 00:08:59.578680 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0327 00:09:23.691211 3878 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0327 00:09:46.572045 3878 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=62 I0327 00:10:04.868610 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:04.869216 3878 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:10:05.707194 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.707239 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576170-9t7hz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.737693 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.737716 3878 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:10:05.743426 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 I0327 00:10:05.743453 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-89xn4 I0327 00:10:28.662193 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0327 00:11:01.677907 3878 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0327 00:11:03.106702 3878 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default] DEL starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default] I0327 00:11:03.189369 3878 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default] DEL finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj 89c24ff8a6a20cdb33801b1aa80bea4daa6d635e4bb852cd657a8b6b650559fd network default NAD default], result "{}", err I0327 00:11:03.248907 3878 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.248967 3878 pods.go:176] Deleting pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj I0327 00:11:03.250460 3878 pods.go:236] Attempting to release IPs for pod: openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj, ips: 10.129.0.62 I0327 00:11:03.250491 3878 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.310697 3878 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.311322 3878 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.366881 3878 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.367479 3878 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 00:11:03.777624 3878 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:03.777649 3878 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:03.785172 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj I0327 00:11:03.785212 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-25hvj I0327 00:11:04.169722 3878 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.169768 3878 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.173208 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq I0327 00:11:04.173244 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-2g6mq I0327 00:11:04.202597 3878 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.202650 3878 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 00:11:04.208163 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j I0327 00:11:04.208191 3878 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-4wd4j